You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ji...@apache.org on 2014/02/14 19:32:46 UTC
svn commit: r1568437 [1/2] - in
/hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop...
Author: jing9
Date: Fri Feb 14 18:32:37 2014
New Revision: 1568437
URL: http://svn.apache.org/r1568437
Log:
Merging r1566359 through r1568420 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
- copied unchanged from r1568420, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
- copied unchanged from r1568420, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
- copied unchanged from r1568420, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TimelineUtils.java
- copied unchanged from r1568420, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TimelineUtils.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/authorize/RMPolicyProvider.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt Fri Feb 14 18:32:37 2014
@@ -6,20 +6,6 @@ Trunk - Unreleased
NEW FEATURES
- YARN-1496. Protocol additions to allow moving apps between queues (Sandy
- Ryza)
-
- YARN-1498. Common scheduler changes for moving apps between queues (Sandy
- Ryza)
-
- YARN-1504. RM changes for moving apps between queues (Sandy Ryza)
-
- YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy
- Ryza)
-
- YARN-1497. Command line additions for moving apps between queues (Sandy
- Ryza)
-
IMPROVEMENTS
OPTIMIZATIONS
@@ -119,6 +105,23 @@ Release 2.4.0 - UNRELEASED
YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. (Billie
Rinaldi via zjshen)
+ YARN-1637. Implemented a client library for Java users to post timeline
+ entities and events. (zjshen)
+
+ YARN-1496. Protocol additions to allow moving apps between queues (Sandy
+ Ryza)
+
+ YARN-1498. Common scheduler changes for moving apps between queues (Sandy
+ Ryza)
+
+ YARN-1504. RM changes for moving apps between queues (Sandy Ryza)
+
+ YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy
+ Ryza)
+
+ YARN-1497. Command line additions for moving apps between queues (Sandy
+ Ryza)
+
IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
@@ -166,6 +169,25 @@ Release 2.4.0 - UNRELEASED
YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
app-attempts separately from apps. (Jian He via vinodkv)
+ YARN-1459. Changed ResourceManager to depend its service initialization
+ on the configuration-provider mechanism during startup too. (Xuan Gong via
+ vinodkv)
+
+ YARN-1706. Created an utility method to dump timeline records to JSON
+ strings. (zjshen)
+
+ YARN-1641. ZK store should attempt a write periodically to ensure it is
+ still Active. (kasha)
+
+ YARN-1531. True up yarn command documentation (Akira Ajisaka via kasha)
+
+ YARN-1345. Remove FINAL_SAVING state from YarnApplicationAttemptState
+ (Zhijie Shen via jianhe)
+
+ YARN-1676. Modified RM HA handling of user-to-group mappings to
+ be available across RM failover by making using of a remote
+ configuration-provider. (Xuan Gong via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -233,7 +255,37 @@ Release 2.4.0 - UNRELEASED
YARN-1672. YarnConfiguration is missing a default for
yarn.nodemanager.log.retain-seconds (Naren Koneru via kasha)
-Release 2.3.0 - UNRELEASED
+ YARN-1698. Fixed default TimelineStore in code to match what is documented
+ in yarn-default.xml (Zhijie Shen via vinodkv)
+
+ YARN-1697. NodeManager reports negative running containers (Sandy Ryza)
+
+ YARN-1719. Fixed the root path related Jersey warnings produced in
+ ATSWebServices. (Billie Rinaldi via zjshen)
+
+ YARN-1692. ConcurrentModificationException in fair scheduler AppSchedulable
+ (Sangjin Lee via Sandy Ryza)
+
+ YARN-1578. Fixed reading incomplete application attempt and container data
+ in FileSystemApplicationHistoryStore. (Shinichi Yamashita via zjshen)
+
+ YARN-1417. Modified RM to generate container-tokens not at creation time, but
+ at allocation time so as to prevent RM from shelling out containers with
+ expired tokens. (Omkar Vinit Joshi and Jian He via vinodkv)
+
+Release 2.3.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+Release 2.3.0 - 2014-02-18
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Fri Feb 14 18:32:37 2014
@@ -309,4 +309,10 @@
<Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+
+ <!-- Multithreaded correctness warnings need to be ignored here as this is for creating the singleton.-->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider"/>
+ <Bug pattern="DC_DOUBLECHECK" />
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java Fri Feb 14 18:32:37 2014
@@ -51,9 +51,6 @@ public enum YarnApplicationAttemptState
/** AppAttempt is currently running. */
RUNNING,
- /** AppAttempt is waiting for state bing saved */
- FINAL_SAVING,
-
/** AppAttempt is finishing. */
FINISHING,
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java Fri Feb 14 18:32:37 2014
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.conf;
import java.io.IOException;
-
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -34,8 +33,8 @@ import org.apache.hadoop.yarn.exceptions
*/
public abstract class ConfigurationProvider {
- public void init(Configuration conf) throws Exception {
- initInternal(conf);
+ public void init(Configuration bootstrapConf) throws Exception {
+ initInternal(bootstrapConf);
}
public void close() throws Exception {
@@ -43,19 +42,21 @@ public abstract class ConfigurationProvi
}
/**
- * Get the configuration.
+ * Get the configuration and combine with bootstrapConf
+ * @param bootstrapConf Configuration
* @param name The configuration file name
* @return configuration
* @throws YarnException
* @throws IOException
*/
- public abstract Configuration getConfiguration(String name)
- throws YarnException, IOException;
+ public abstract Configuration getConfiguration(Configuration bootstrapConf,
+ String name) throws YarnException, IOException;
/**
* Derived classes initialize themselves using this method.
*/
- public abstract void initInternal(Configuration conf) throws Exception;
+ public abstract void initInternal(Configuration bootstrapConf)
+ throws Exception;
/**
* Derived classes close themselves using this method.
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java Fri Feb 14 18:32:37 2014
@@ -33,12 +33,12 @@ public class ConfigurationProviderFactor
/**
* Creates an instance of {@link ConfigurationProvider} using given
* configuration.
- * @param conf
+ * @param bootstrapConf
* @return configurationProvider
*/
@SuppressWarnings("unchecked")
public static ConfigurationProvider
- getConfigurationProvider(Configuration conf) {
+ getConfigurationProvider(Configuration bootstrapConf) {
Class<? extends ConfigurationProvider> defaultProviderClass;
try {
defaultProviderClass = (Class<? extends ConfigurationProvider>)
@@ -49,9 +49,11 @@ public class ConfigurationProviderFactor
"Invalid default configuration provider class"
+ YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS, e);
}
- ConfigurationProvider configurationProvider = ReflectionUtils.newInstance(
- conf.getClass(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- defaultProviderClass, ConfigurationProvider.class), conf);
+ ConfigurationProvider configurationProvider =
+ ReflectionUtils.newInstance(bootstrapConf.getClass(
+ YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ defaultProviderClass, ConfigurationProvider.class),
+ bootstrapConf);
return configurationProvider;
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Feb 14 18:32:37 2014
@@ -120,10 +120,9 @@ enum YarnApplicationAttemptStateProto {
APP_ATTEMPT_LAUNCHED = 6;
APP_ATTEMPT_FAILED = 7;
APP_ATTEMPT_RUNNING = 8;
- APP_ATTEMPT_FINAL_SAVING = 9;
- APP_ATTEMPT_FINISHING = 10;
- APP_ATTEMPT_FINISHED = 11;
- APP_ATTEMPT_KILLED = 12;
+ APP_ATTEMPT_FINISHING = 9;
+ APP_ATTEMPT_FINISHED = 10;
+ APP_ATTEMPT_KILLED = 11;
}
enum FinalApplicationStatusProto {
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Fri Feb 14 18:32:37 2014
@@ -79,6 +79,10 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Fri Feb 14 18:32:37 2014
@@ -382,11 +382,7 @@ public class ApplicationCLI extends Yarn
}
/**
- * Kills the application with the application id as appId
- *
- * @param applicationId
- * @throws YarnException
- * @throws IOException
+ * Moves the application with the given ID to the given queue.
*/
private void moveApplicationAcrossQueues(String applicationId, String queue)
throws YarnException, IOException {
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Fri Feb 14 18:32:37 2014
@@ -675,7 +675,6 @@ public class TestYarnCLI {
int result = spyCli.run(new String[] { "-help" });
Assert.assertTrue(result == 0);
verify(spyCli).printUsage(any(Options.class));
- System.err.println(sysOutStream.toString()); //todo sandyt remove this hejfkdsl
Assert.assertEquals(createApplicationCLIHelpMessage(),
sysOutStream.toString());
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java Fri Feb 14 18:32:37 2014
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn;
import java.io.IOException;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -42,24 +41,24 @@ public class FileSystemBasedConfiguratio
private Path configDir;
@Override
- public synchronized Configuration getConfiguration(String name)
- throws IOException, YarnException {
+ public synchronized Configuration getConfiguration(Configuration bootstrapConf,
+ String name) throws IOException, YarnException {
Path configPath = new Path(this.configDir, name);
if (!fs.exists(configPath)) {
throw new YarnException("Can not find Configuration: " + name + " in "
+ configDir);
}
- Configuration conf = new Configuration(false);
- conf.addResource(fs.open(configPath));
- return conf;
+ bootstrapConf.addResource(fs.open(configPath));
+ return bootstrapConf;
}
@Override
- public synchronized void initInternal(Configuration conf) throws Exception {
+ public synchronized void initInternal(Configuration bootstrapConf)
+ throws Exception {
configDir =
- new Path(conf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE,
+ new Path(bootstrapConf.get(YarnConfiguration.FS_BASED_RM_CONF_STORE,
YarnConfiguration.DEFAULT_FS_BASED_RM_CONF_STORE));
- fs = configDir.getFileSystem(conf);
+ fs = configDir.getFileSystem(bootstrapConf);
if (!fs.exists(configDir)) {
fs.mkdirs(configDir);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java Fri Feb 14 18:32:37 2014
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn;
import java.io.IOException;
-
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -31,13 +30,13 @@ import org.apache.hadoop.yarn.exceptions
public class LocalConfigurationProvider extends ConfigurationProvider {
@Override
- public Configuration getConfiguration(String name)
- throws IOException, YarnException {
- return new Configuration();
+ public Configuration getConfiguration(Configuration bootstrapConf,
+ String name) throws IOException, YarnException {
+ return bootstrapConf;
}
@Override
- public void initInternal(Configuration conf) throws Exception {
+ public void initInternal(Configuration bootstrapConf) throws Exception {
// Do nothing
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/TestApplicationTimelineRecords.java Fri Feb 14 18:32:37 2014
@@ -19,18 +19,23 @@
package org.apache.hadoop.yarn.api.records.apptimeline;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import junit.framework.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors.ATSPutError;
+import org.apache.hadoop.yarn.util.TimelineUtils;
import org.junit.Test;
public class TestApplicationTimelineRecords {
+ private static final Log LOG =
+ LogFactory.getLog(TestApplicationTimelineRecords.class);
+
@Test
- public void testATSEntities() {
+ public void testATSEntities() throws Exception {
ATSEntities entities = new ATSEntities();
for (int j = 0; j < 2; ++j) {
ATSEntity entity = new ATSEntity();
@@ -53,6 +58,9 @@ public class TestApplicationTimelineReco
entity.addOtherInfo("okey2", "oval2");
entities.addEntity(entity);
}
+ LOG.info("Entities in JSON:");
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(entities, true));
+
Assert.assertEquals(2, entities.getEntities().size());
ATSEntity entity1 = entities.getEntities().get(0);
Assert.assertEquals("entity id 0", entity1.getEntityId());
@@ -71,7 +79,7 @@ public class TestApplicationTimelineReco
}
@Test
- public void testATSEvents() {
+ public void testATSEvents() throws Exception {
ATSEvents events = new ATSEvents();
for (int j = 0; j < 2; ++j) {
ATSEvents.ATSEventsOfOneEntity partEvents =
@@ -88,6 +96,9 @@ public class TestApplicationTimelineReco
}
events.addEvent(partEvents);
}
+ LOG.info("Events in JSON:");
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(events, true));
+
Assert.assertEquals(2, events.getAllEvents().size());
ATSEvents.ATSEventsOfOneEntity partEvents1 = events.getAllEvents().get(0);
Assert.assertEquals("entity id 0", partEvents1.getEntityId());
@@ -112,7 +123,7 @@ public class TestApplicationTimelineReco
}
@Test
- public void testATSPutErrors() {
+ public void testATSPutErrors() throws Exception {
ATSPutErrors atsPutErrors = new ATSPutErrors();
ATSPutError error1 = new ATSPutError();
error1.setEntityId("entity id 1");
@@ -127,6 +138,8 @@ public class TestApplicationTimelineReco
error2.setErrorCode(ATSPutError.IO_EXCEPTION);
errors.add(error2);
atsPutErrors.addErrors(errors);
+ LOG.info("Errors in JSON:");
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(atsPutErrors, true));
Assert.assertEquals(3, atsPutErrors.getErrors().size());
ATSPutError e = atsPutErrors.getErrors().get(0);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Fri Feb 14 18:32:37 2014
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.YarnUncaug
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.MemoryApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -143,10 +143,8 @@ public class ApplicationHistoryServer ex
protected ApplicationTimelineStore createApplicationTimelineStore(
Configuration conf) {
- // TODO: need to replace the MemoryApplicationTimelineStore.class with the
- // LevelDB implementation
return ReflectionUtils.newInstance(conf.getClass(
- YarnConfiguration.ATS_STORE, MemoryApplicationTimelineStore.class,
+ YarnConfiguration.ATS_STORE, LeveldbApplicationTimelineStore.class,
ApplicationTimelineStore.class), conf);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Fri Feb 14 18:32:37 2014
@@ -215,17 +215,30 @@ public class FileSystemApplicationHistor
getApplicationAttempts(ApplicationId appId) throws IOException {
Map<ApplicationAttemptId, ApplicationAttemptHistoryData> historyDataMap =
new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
- Map<ApplicationAttemptId, StartFinishDataPair<ApplicationAttemptStartData, ApplicationAttemptFinishData>> startFinshDataMap =
- new HashMap<ApplicationAttemptId, StartFinishDataPair<ApplicationAttemptStartData, ApplicationAttemptFinishData>>();
HistoryFileReader hfReader = getHistoryFileReader(appId);
try {
while (hfReader.hasNext()) {
HistoryFileReader.Entry entry = hfReader.next();
- if (entry.key.id.startsWith(ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
- if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
- retrieveStartFinishData(appId, entry, startFinshDataMap, true);
- } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
- retrieveStartFinishData(appId, entry, startFinshDataMap, false);
+ if (entry.key.id.startsWith(
+ ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
+ ApplicationAttemptId appAttemptId =
+ ConverterUtils.toApplicationAttemptId(entry.key.id);
+ if (appAttemptId.getApplicationId().equals(appId)) {
+ ApplicationAttemptHistoryData historyData =
+ historyDataMap.get(appAttemptId);
+ if (historyData == null) {
+ historyData = ApplicationAttemptHistoryData.newInstance(
+ appAttemptId, null, -1, null, null, null,
+ FinalApplicationStatus.UNDEFINED, null);
+ historyDataMap.put(appAttemptId, historyData);
+ }
+ if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
+ mergeApplicationAttemptHistoryData(historyData,
+ parseApplicationAttemptStartData(entry.value));
+ } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
+ mergeApplicationAttemptHistoryData(historyData,
+ parseApplicationAttemptFinishData(entry.value));
+ }
}
}
}
@@ -237,45 +250,9 @@ public class FileSystemApplicationHistor
} finally {
hfReader.close();
}
- for (Map.Entry<ApplicationAttemptId, StartFinishDataPair<ApplicationAttemptStartData, ApplicationAttemptFinishData>> entry : startFinshDataMap
- .entrySet()) {
- ApplicationAttemptHistoryData historyData =
- ApplicationAttemptHistoryData.newInstance(entry.getKey(), null, -1,
- null, null, null, FinalApplicationStatus.UNDEFINED, null);
- mergeApplicationAttemptHistoryData(historyData,
- entry.getValue().startData);
- mergeApplicationAttemptHistoryData(historyData,
- entry.getValue().finishData);
- historyDataMap.put(entry.getKey(), historyData);
- }
return historyDataMap;
}
- private
- void
- retrieveStartFinishData(
- ApplicationId appId,
- HistoryFileReader.Entry entry,
- Map<ApplicationAttemptId, StartFinishDataPair<ApplicationAttemptStartData, ApplicationAttemptFinishData>> startFinshDataMap,
- boolean start) throws IOException {
- ApplicationAttemptId appAttemptId =
- ConverterUtils.toApplicationAttemptId(entry.key.id);
- if (appAttemptId.getApplicationId().equals(appId)) {
- StartFinishDataPair<ApplicationAttemptStartData, ApplicationAttemptFinishData> pair =
- startFinshDataMap.get(appAttemptId);
- if (pair == null) {
- pair =
- new StartFinishDataPair<ApplicationAttemptStartData, ApplicationAttemptFinishData>();
- startFinshDataMap.put(appAttemptId, pair);
- }
- if (start) {
- pair.startData = parseApplicationAttemptStartData(entry.value);
- } else {
- pair.finishData = parseApplicationAttemptFinishData(entry.value);
- }
- }
- }
-
@Override
public ApplicationAttemptHistoryData getApplicationAttempt(
ApplicationAttemptId appAttemptId) throws IOException {
@@ -391,20 +368,30 @@ public class FileSystemApplicationHistor
ApplicationAttemptId appAttemptId) throws IOException {
Map<ContainerId, ContainerHistoryData> historyDataMap =
new HashMap<ContainerId, ContainerHistoryData>();
- Map<ContainerId, StartFinishDataPair<ContainerStartData, ContainerFinishData>> startFinshDataMap =
- new HashMap<ContainerId, StartFinishDataPair<ContainerStartData, ContainerFinishData>>();
HistoryFileReader hfReader =
getHistoryFileReader(appAttemptId.getApplicationId());
try {
while (hfReader.hasNext()) {
HistoryFileReader.Entry entry = hfReader.next();
if (entry.key.id.startsWith(ConverterUtils.CONTAINER_PREFIX)) {
- if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
- retrieveStartFinishData(appAttemptId, entry, startFinshDataMap,
- true);
- } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
- retrieveStartFinishData(appAttemptId, entry, startFinshDataMap,
- false);
+ ContainerId containerId =
+ ConverterUtils.toContainerId(entry.key.id);
+ if (containerId.getApplicationAttemptId().equals(appAttemptId)) {
+ ContainerHistoryData historyData =
+ historyDataMap.get(containerId);
+ if (historyData == null) {
+ historyData = ContainerHistoryData.newInstance(
+ containerId, null, null, null, Long.MIN_VALUE,
+ Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null);
+ historyDataMap.put(containerId, historyData);
+ }
+ if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
+ mergeContainerHistoryData(historyData,
+ parseContainerStartData(entry.value));
+ } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
+ mergeContainerHistoryData(historyData,
+ parseContainerFinishData(entry.value));
+ }
}
}
}
@@ -416,43 +403,9 @@ public class FileSystemApplicationHistor
} finally {
hfReader.close();
}
- for (Map.Entry<ContainerId, StartFinishDataPair<ContainerStartData, ContainerFinishData>> entry : startFinshDataMap
- .entrySet()) {
- ContainerHistoryData historyData =
- ContainerHistoryData
- .newInstance(entry.getKey(), null, null, null, Long.MIN_VALUE,
- Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null);
- mergeContainerHistoryData(historyData, entry.getValue().startData);
- mergeContainerHistoryData(historyData, entry.getValue().finishData);
- historyDataMap.put(entry.getKey(), historyData);
- }
return historyDataMap;
}
- private
- void
- retrieveStartFinishData(
- ApplicationAttemptId appAttemptId,
- HistoryFileReader.Entry entry,
- Map<ContainerId, StartFinishDataPair<ContainerStartData, ContainerFinishData>> startFinshDataMap,
- boolean start) throws IOException {
- ContainerId containerId = ConverterUtils.toContainerId(entry.key.id);
- if (containerId.getApplicationAttemptId().equals(appAttemptId)) {
- StartFinishDataPair<ContainerStartData, ContainerFinishData> pair =
- startFinshDataMap.get(containerId);
- if (pair == null) {
- pair =
- new StartFinishDataPair<ContainerStartData, ContainerFinishData>();
- startFinshDataMap.put(containerId, pair);
- }
- if (start) {
- pair.startData = parseContainerStartData(entry.value);
- } else {
- pair.finishData = parseContainerFinishData(entry.value);
- }
- }
- }
-
@Override
public void applicationStarted(ApplicationStartData appStart)
throws IOException {
@@ -828,14 +781,5 @@ public class FileSystemApplicationHistor
id = in.readUTF();
suffix = in.readUTF();
}
-
}
-
- private static class StartFinishDataPair<S, F> {
-
- private S startData;
- private F finishData;
-
- }
-
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java Fri Feb 14 18:32:37 2014
@@ -107,7 +107,6 @@ public class ATSWebServices {
* Return the description of the application timeline web services.
*/
@GET
- @Path("/")
@Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
public AboutInfo about(
@Context HttpServletRequest req,
@@ -235,7 +234,6 @@ public class ATSWebServices {
* that happen during storing.
*/
@POST
- @Path("/")
@Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
public ATSPutErrors postEntities(
@Context HttpServletRequest req,
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Fri Feb 14 18:32:37 2014
@@ -72,6 +72,12 @@ public class TestFileSystemApplicationHi
}
private void testWriteHistoryData(int num) throws IOException {
+ testWriteHistoryData(num, false, false);
+ }
+
+ private void testWriteHistoryData(
+ int num, boolean missingContainer, boolean missingApplicationAttempt)
+ throws IOException {
// write application history data
for (int i = 1; i <= num; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i);
@@ -83,21 +89,31 @@ public class TestFileSystemApplicationHi
ApplicationAttemptId.newInstance(appId, j);
writeApplicationAttemptStartData(appAttemptId);
+ if (missingApplicationAttempt && j == num) {
+ continue;
+ }
// write container history data
for (int k = 1; k <= num; ++k) {
ContainerId containerId = ContainerId.newInstance(appAttemptId, k);
writeContainerStartData(containerId);
+ if (missingContainer && k == num) {
+ continue;
+ }
writeContainerFinishData(containerId);
-
- writeApplicationAttemptFinishData(appAttemptId);
}
+ writeApplicationAttemptFinishData(appAttemptId);
}
-
writeApplicationFinishData(appId);
}
}
private void testReadHistoryData(int num) throws IOException {
+ testReadHistoryData(num, false, false);
+ }
+
+ private void testReadHistoryData(
+ int num, boolean missingContainer, boolean missingApplicationAttempt)
+ throws IOException {
// read application history data
Assert.assertEquals(num, store.getAllApplications().size());
for (int i = 1; i <= num; ++i) {
@@ -116,8 +132,14 @@ public class TestFileSystemApplicationHi
store.getApplicationAttempt(appAttemptId);
Assert.assertNotNull(attemptData);
Assert.assertEquals(appAttemptId.toString(), attemptData.getHost());
- Assert.assertEquals(appAttemptId.toString(),
- attemptData.getDiagnosticsInfo());
+
+ if (missingApplicationAttempt && j == num) {
+ Assert.assertNull(attemptData.getDiagnosticsInfo());
+ continue;
+ } else {
+ Assert.assertEquals(appAttemptId.toString(),
+ attemptData.getDiagnosticsInfo());
+ }
// read container history data
Assert.assertEquals(num, store.getContainers(appAttemptId).size());
@@ -127,8 +149,12 @@ public class TestFileSystemApplicationHi
Assert.assertNotNull(containerData);
Assert.assertEquals(Priority.newInstance(containerId.getId()),
containerData.getPriority());
- Assert.assertEquals(containerId.toString(),
- containerData.getDiagnosticsInfo());
+ if (missingContainer && k == num) {
+ Assert.assertNull(containerData.getDiagnosticsInfo());
+ } else {
+ Assert.assertEquals(containerId.toString(),
+ containerData.getDiagnosticsInfo());
+ }
}
ContainerHistoryData masterContainer =
store.getAMContainer(appAttemptId);
@@ -193,4 +219,15 @@ public class TestFileSystemApplicationHi
Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20);
}
+ @Test
+ public void testMissingContainerHistoryData() throws IOException {
+ testWriteHistoryData(3, true, false);
+ testReadHistoryData(3, true, false);
+ }
+
+ @Test
+ public void testMissingApplicationAttemptHistoryData() throws IOException {
+ testWriteHistoryData(3, false, true);
+ testReadHistoryData(3, false, true);
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri Feb 14 18:32:37 2014
@@ -83,6 +83,7 @@ public class ContainerImpl implements Co
private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
+ private boolean wasLaunched;
/** The NM-wide configuration - not specific to this container */
private final Configuration daemonConf;
@@ -418,7 +419,9 @@ public class ContainerImpl implements Co
applicationId, containerId);
break;
case EXITED_WITH_FAILURE:
- metrics.endRunningContainer();
+ if (wasLaunched) {
+ metrics.endRunningContainer();
+ }
// fall through
case LOCALIZATION_FAILED:
metrics.failedContainer();
@@ -428,7 +431,9 @@ public class ContainerImpl implements Co
applicationId, containerId);
break;
case CONTAINER_CLEANEDUP_AFTER_KILL:
- metrics.endRunningContainer();
+ if (wasLaunched) {
+ metrics.endRunningContainer();
+ }
// fall through
case NEW:
metrics.killedContainer();
@@ -636,6 +641,7 @@ public class ContainerImpl implements Co
new ContainerStartMonitoringEvent(container.containerId,
vmemBytes, pmemBytes));
container.metrics.runningContainer();
+ container.wasLaunched = true;
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java Fri Feb 14 18:32:37 2014
@@ -99,4 +99,8 @@ public class NodeManagerMetrics {
public void addResource(Resource res) {
availableGB.incr(res.getMemory() / 1024);
}
+
+ public int getRunningContainers() {
+ return containersRunning.value();
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Fri Feb 14 18:32:37 2014
@@ -348,6 +348,9 @@ public class TestContainer {
wc.c.getContainerState());
assertNull(wc.c.getLocalizedResources());
verifyCleanupCall(wc);
+ wc.c.handle(new ContainerEvent(wc.c.getContainerId(),
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+ assertEquals(0, metrics.getRunningContainers());
} finally {
if (wc != null) {
wc.finished();
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Fri Feb 14 18:32:37 2014
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceStatus;
@@ -45,11 +46,8 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
-import org.apache.hadoop.yarn.conf.ConfigurationProvider;
-import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -92,8 +90,6 @@ public class AdminService extends Compos
private Server server;
private InetSocketAddress masterServiceAddress;
private AccessControlList adminAcl;
-
- private ConfigurationProvider configurationProvider = null;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -115,10 +111,6 @@ public class AdminService extends Compos
}
}
- this.configurationProvider =
- ConfigurationProviderFactory.getConfigurationProvider(conf);
- configurationProvider.init(conf);
-
masterServiceAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
@@ -139,9 +131,6 @@ public class AdminService extends Compos
@Override
protected synchronized void serviceStop() throws Exception {
stopServer();
- if (this.configurationProvider != null) {
- configurationProvider.close();
- }
super.serviceStop();
}
@@ -158,7 +147,10 @@ public class AdminService extends Compos
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(conf, new RMPolicyProvider());
+ refreshServiceAcls(
+ getConfiguration(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
+ RMPolicyProvider.getInstance());
}
if (rmContext.isHAEnabled()) {
@@ -321,8 +313,8 @@ public class AdminService extends Compos
RefreshQueuesResponse response =
recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
- Configuration conf =
- getConfiguration(YarnConfiguration.CS_CONFIGURATION_FILE);
+ Configuration conf = getConfiguration(getConfig(),
+ YarnConfiguration.CS_CONFIGURATION_FILE);
rmContext.getScheduler().reinitialize(conf, this.rmContext);
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
@@ -376,7 +368,8 @@ public class AdminService extends Compos
}
Configuration conf =
- getConfiguration(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ getConfiguration(getConfig(),
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
RMAuditLogger.logSuccess(user.getShortUserName(),
argName, "AdminService");
@@ -388,21 +381,22 @@ public class AdminService extends Compos
@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
RefreshUserToGroupsMappingsRequest request)
- throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshUserToGroupsMappings");
+ throws YarnException, IOException {
+ String argName = "refreshUserToGroupsMappings";
+ UserGroupInformation user = checkAcls(argName);
- // TODO (YARN-1459): Revisit handling user-groups on Standby RM
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(),
- "refreshUserToGroupsMapping",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh user-groups.");
throwStandbyException();
}
- Groups.getUserToGroupsMappingService().refresh();
- RMAuditLogger.logSuccess(user.getShortUserName(),
- "refreshUserToGroupsMappings", "AdminService");
+ Groups.getUserToGroupsMappingService(
+ getConfiguration(getConfig(),
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)).refresh();
+
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName, "AdminService");
return recordFactory.newRecordInstance(
RefreshUserToGroupsMappingsResponse.class);
@@ -421,7 +415,7 @@ public class AdminService extends Compos
throwStandbyException();
}
Configuration conf =
- getConfiguration(YarnConfiguration.YARN_SITE_XML_FILE);
+ getConfiguration(getConfig(), YarnConfiguration.YARN_SITE_XML_FILE);
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
@@ -452,9 +446,10 @@ public class AdminService extends Compos
throwStandbyException();
}
- PolicyProvider policyProvider = new RMPolicyProvider();
+ PolicyProvider policyProvider = RMPolicyProvider.getInstance();
Configuration conf =
- getConfiguration(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
+ getConfiguration(getConfig(),
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
refreshServiceAcls(conf, policyProvider);
rmContext.getClientRMService().refreshServiceAcls(conf, policyProvider);
@@ -466,12 +461,13 @@ public class AdminService extends Compos
return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class);
}
- synchronized void refreshServiceAcls(Configuration configuration,
+ private synchronized void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.configurationProvider instanceof LocalConfigurationProvider) {
+ if (this.rmContext.getConfigurationProvider() instanceof
+ LocalConfigurationProvider) {
this.server.refreshServiceAcl(configuration, policyProvider);
} else {
- this.server.refreshServiceAclWithConfigration(configuration,
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
policyProvider);
}
}
@@ -521,9 +517,10 @@ public class AdminService extends Compos
return response;
}
- private synchronized Configuration getConfiguration(String confFileName)
- throws YarnException, IOException {
- return this.configurationProvider.getConfiguration(confFileName);
+ private synchronized Configuration getConfiguration(Configuration conf,
+ String confFileName) throws YarnException, IOException {
+ return this.rmContext.getConfigurationProvider().getConfiguration(conf,
+ confFileName);
}
@VisibleForTesting
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Fri Feb 14 18:32:37 2014
@@ -105,7 +105,6 @@ public class ApplicationMasterService ex
private final AllocateResponse resync =
recordFactory.newRecordInstance(AllocateResponse.class);
private final RMContext rmContext;
- private boolean useLocalConfigurationProvider;
public ApplicationMasterService(RMContext rmContext, YarnScheduler scheduler) {
super(ApplicationMasterService.class.getName());
@@ -116,15 +115,6 @@ public class ApplicationMasterService ex
}
@Override
- protected void serviceInit(Configuration conf) throws Exception {
- this.useLocalConfigurationProvider =
- (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
- YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- LocalConfigurationProvider.class)));
- super.serviceInit(conf);
- }
-
- @Override
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
@@ -150,7 +140,10 @@ public class ApplicationMasterService ex
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(conf, new RMPolicyProvider());
+ refreshServiceAcls(
+ this.rmContext.getConfigurationProvider().getConfiguration(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
+ RMPolicyProvider.getInstance());
}
this.server.start();
@@ -591,10 +584,11 @@ public class ApplicationMasterService ex
public void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.useLocalConfigurationProvider) {
+ if (this.rmContext.getConfigurationProvider() instanceof
+ LocalConfigurationProvider) {
this.server.refreshServiceAcl(configuration, policyProvider);
} else {
- this.server.refreshServiceAclWithConfigration(configuration,
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
policyProvider);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Feb 14 18:32:37 2014
@@ -136,7 +136,6 @@ public class ClientRMService extends Abs
private final ApplicationACLsManager applicationsACLsManager;
private final QueueACLsManager queueACLsManager;
- private boolean useLocalConfigurationProvider;
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
@@ -154,10 +153,6 @@ public class ClientRMService extends Abs
@Override
protected void serviceInit(Configuration conf) throws Exception {
clientBindAddress = getBindAddress(conf);
- this.useLocalConfigurationProvider =
- (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
- YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- LocalConfigurationProvider.class)));
super.serviceInit(conf);
}
@@ -176,7 +171,10 @@ public class ClientRMService extends Abs
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(conf, new RMPolicyProvider());
+ refreshServiceAcls(
+ this.rmContext.getConfigurationProvider().getConfiguration(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
+ RMPolicyProvider.getInstance());
}
this.server.start();
@@ -809,10 +807,11 @@ public class ClientRMService extends Abs
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.useLocalConfigurationProvider) {
+ if (this.rmContext.getConfigurationProvider() instanceof
+ LocalConfigurationProvider) {
this.server.refreshServiceAcl(configuration, policyProvider);
} else {
- this.server.refreshServiceAclWithConfigration(configuration,
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
policyProvider);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Fri Feb 14 18:32:37 2014
@@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentMa
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -97,4 +98,5 @@ public interface RMContext {
void setRMApplicationHistoryWriter(
RMApplicationHistoryWriter rmApplicationHistoryWriter);
+ ConfigurationProvider getConfigurationProvider();
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Fri Feb 14 18:32:37 2014
@@ -23,8 +23,10 @@ import java.util.concurrent.ConcurrentMa
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
@@ -78,7 +80,7 @@ public class RMContextImpl implements RM
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
private RMApplicationHistoryWriter rmApplicationHistoryWriter;
-
+ private ConfigurationProvider configurationProvider;
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -119,8 +121,11 @@ public class RMContextImpl implements RM
} catch (Exception e) {
assert false;
}
+
+ ConfigurationProvider provider = new LocalConfigurationProvider();
+ setConfigurationProvider(provider);
}
-
+
@Override
public Dispatcher getDispatcher() {
return this.rmDispatcher;
@@ -334,4 +339,13 @@ public class RMContextImpl implements RM
this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
}
+ @Override
+ public ConfigurationProvider getConfigurationProvider() {
+ return this.configurationProvider;
+ }
+
+ public void setConfigurationProvider(
+ ConfigurationProvider configurationProvider) {
+ this.configurationProvider = configurationProvider;
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java Fri Feb 14 18:32:37 2014
@@ -214,8 +214,6 @@ public class RMServerUtils {
return YarnApplicationAttemptState.RUNNING;
case FINISHING:
return YarnApplicationAttemptState.FINISHING;
- case FINAL_SAVING:
- return YarnApplicationAttemptState.FINAL_SAVING;
case FINISHED:
return YarnApplicationAttemptState.FINISHED;
case KILLED:
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri Feb 14 18:32:37 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.http.HttpConfig
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
@@ -42,10 +43,13 @@ import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -154,7 +158,7 @@ public class ResourceManager extends Com
private boolean recoveryEnabled;
private String webAppAddress;
-
+ private ConfigurationProvider configurationProvider = null;
/** End of Active services */
private Configuration conf;
@@ -182,6 +186,24 @@ public class ResourceManager extends Com
this.conf = conf;
this.rmContext = new RMContextImpl();
+ this.configurationProvider =
+ ConfigurationProviderFactory.getConfigurationProvider(conf);
+ this.configurationProvider.init(this.conf);
+ rmContext.setConfigurationProvider(configurationProvider);
+ if (!(this.configurationProvider instanceof LocalConfigurationProvider)) {
+ // load yarn-site.xml
+ this.conf =
+ this.configurationProvider.getConfiguration(this.conf,
+ YarnConfiguration.YARN_SITE_XML_FILE);
+ // load core-site.xml
+ this.conf =
+ this.configurationProvider.getConfiguration(this.conf,
+ YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ // Do refreshUserToGroupsMappings with loaded core-site.xml
+ Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
+ .refresh();
+ }
+
// register the handlers for all AlwaysOn services using setupDispatcher().
rmDispatcher = setupDispatcher();
addIfService(rmDispatcher);
@@ -884,6 +906,9 @@ public class ResourceManager extends Com
if (fetcher != null) {
fetcher.stop();
}
+ if (configurationProvider != null) {
+ configurationProvider.close();
+ }
super.serviceStop();
transitionToStandby(false);
rmContext.setHAServiceState(HAServiceState.STOPPING);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Fri Feb 14 18:32:37 2014
@@ -95,7 +95,6 @@ public class ResourceTrackerService exte
private int minAllocMb;
private int minAllocVcores;
- private boolean useLocalConfigurationProvider;
static {
resync.setNodeAction(NodeAction.RESYNC);
@@ -145,10 +144,6 @@ public class ResourceTrackerService exte
YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);
- this.useLocalConfigurationProvider =
- (LocalConfigurationProvider.class.isAssignableFrom(conf.getClass(
- YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- LocalConfigurationProvider.class)));
super.serviceInit(conf);
}
@@ -169,7 +164,10 @@ public class ResourceTrackerService exte
if (conf.getBoolean(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
false)) {
- refreshServiceAcls(conf, new RMPolicyProvider());
+ refreshServiceAcls(
+ this.rmContext.getConfigurationProvider().getConfiguration(conf,
+ YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
+ RMPolicyProvider.getInstance());
}
this.server.start();
@@ -423,10 +421,11 @@ public class ResourceTrackerService exte
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
- if (this.useLocalConfigurationProvider) {
+ if (this.rmContext.getConfigurationProvider() instanceof
+ LocalConfigurationProvider) {
this.server.refreshServiceAcl(configuration, policyProvider);
} else {
- this.server.refreshServiceAclWithConfigration(configuration,
+ this.server.refreshServiceAclWithLoadedConfiguration(configuration,
policyProvider);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Fri Feb 14 18:32:37 2014
@@ -676,11 +676,11 @@ public abstract class RMStateStore exten
@SuppressWarnings("unchecked")
/**
- * In {#handleStoreEvent}, this method is called to notify the
- * ResourceManager that the store operation has failed.
+ * This method is called to notify the ResourceManager that the store
+ * operation has failed.
* @param failureCause the exception due to which the operation failed
*/
- private void notifyStoreOperationFailed(Exception failureCause) {
+ protected void notifyStoreOperationFailed(Exception failureCause) {
RMFatalEventType type;
if (failureCause instanceof StoreFencedException) {
type = RMFatalEventType.STATE_STORE_FENCED;
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1568437&r1=1568436&r2=1568437&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Fri Feb 14 18:32:37 2014
@@ -137,6 +137,7 @@ public class ZKRMStateStore extends RMSt
private String fencingNodePath;
private Op createFencingNodePathOp;
private Op deleteFencingNodePathOp;
+ private Thread verifyActiveStatusThread;
private String zkRootNodeUsername;
private final String zkRootNodePassword = Long.toString(random.nextLong());
@@ -258,6 +259,8 @@ public class ZKRMStateStore extends RMSt
createRootDir(zkRootNodePath);
if (HAUtil.isHAEnabled(getConfig())){
fence();
+ verifyActiveStatusThread = new VerifyActiveStatusThread();
+ verifyActiveStatusThread.start();
}
createRootDir(rmAppRoot);
createRootDir(rmDTSecretManagerRoot);
@@ -350,6 +353,10 @@ public class ZKRMStateStore extends RMSt
@Override
protected synchronized void closeInternal() throws Exception {
+ if (verifyActiveStatusThread != null) {
+ verifyActiveStatusThread.interrupt();
+ verifyActiveStatusThread.join(1000);
+ }
closeZkClients();
}
@@ -856,6 +863,32 @@ public class ZKRMStateStore extends RMSt
}.runWithRetries();
}
+ /**
+ * Helper class that periodically attempts creating a znode to ensure that
+ * this RM continues to be the Active.
+ */
+ private class VerifyActiveStatusThread extends Thread {
+ private List<Op> emptyOpList = new ArrayList<Op>();
+
+ VerifyActiveStatusThread() {
+ super(VerifyActiveStatusThread.class.getName());
+ }
+
+ public void run() {
+ try {
+ while (true) {
+ doMultiWithRetries(emptyOpList);
+ Thread.sleep(zkSessionTimeout);
+ }
+ } catch (InterruptedException ie) {
+ LOG.info(VerifyActiveStatusThread.class.getName() + " thread " +
+ "interrupted! Exiting!");
+ } catch (Exception e) {
+ notifyStoreOperationFailed(new StoreFencedException());
+ }
+ }
+ }
+
private abstract class ZKAction<T> {
// run() expects synchronization on ZKRMStateStore.this
abstract T run() throws KeeperException, InterruptedException;