You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2014/02/25 02:16:32 UTC
svn commit: r1571509 - in
/hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/t...
Author: szetszwo
Date: Tue Feb 25 01:16:30 2014
New Revision: 1571509
URL: http://svn.apache.org/r1571509
Log:
Merge r1569890 through r1571508 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/
- copied from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/
- copied from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/
- copied from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/
- copied from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
- copied unchanged from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TimelineWebServices.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/
- copied from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/timeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java
- copied unchanged from r1571508, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java
Removed:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/apptimeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/TimelineUtils.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/apptimeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/ATSWebServices.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/apptimeline/
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestATSWebServices.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt Tue Feb 25 01:16:30 2014
@@ -28,6 +28,8 @@ Release 2.5.0 - UNRELEASED
YARN-1736. FS: AppSchedulable.assignContainer's priority argument is
redundant. (Naren Koneru via kasha)
+ YARN-1678. Fair scheduler gabs incessantly about reservations (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -218,6 +220,15 @@ Release 2.4.0 - UNRELEASED
YARN-1470. Add audience annotations to MiniYARNCluster. (Anubhav Dhoot
via kasha)
+ YARN-1732. Changed types of related-entities and primary-filters in the
+ timeline-service to be sets instead of maps. (Billie Rinaldi via vinodkv)
+
+ YARN-1687. Renamed user-facing records for the timeline-service to be simply
+ named after 'timeline' instead of 'apptimeline'. (Zhijie Shen via vinodkv)
+
+ YARN-1749. Updated application-history related configs to reflect the latest
+ reality and to be consistently named. (Zhijie Shen via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -330,6 +341,13 @@ Release 2.4.0 - UNRELEASED
YARN-1071. Enabled ResourceManager to recover cluster metrics
numDecommissionedNMs after restarting. (Jian He via zjshen)
+ YARN-1742. Fixed javadoc of configuration parameter
+ DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION. (Akira Ajisaka via vinodkv)
+
+ YARN-1686. Fixed NodeManager to properly handle any errors during
+ re-registration after a RESYNC and thus avoid hanging. (Rohith Sharma via
+ vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Feb 25 01:16:30 2014
@@ -300,11 +300,6 @@ public class YarnConfiguration extends C
public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
10;
- /** The implementation class of ApplicationHistoryStore, which is to be used
- * by RMApplicationHistoryWriter. */
- public static final String RM_HISTORY_WRITER_CLASS = RM_PREFIX
- + "history-writer.class";
-
//Delegation token related keys
public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";
@@ -741,7 +736,7 @@ public class YarnConfiguration extends C
public static final String NM_MIN_HEALTHY_DISKS_FRACTION =
NM_PREFIX + "disk-health-checker.min-healthy-disks";
/**
- * By default, at least 5% of disks are to be healthy to say that the node
+ * By default, at least 25% of disks are to be healthy to say that the node
* is healthy in terms of disks.
*/
public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION
@@ -994,74 +989,91 @@ public class YarnConfiguration extends C
YARN_PREFIX + "app.container.log.backups";
////////////////////////////////
- // AHS Configs
+ // Timeline Service Configs
////////////////////////////////
- public static final String AHS_PREFIX = YARN_PREFIX + "ahs.";
-
- /** The setting that controls whether history-service is enabled or not.. */
- public static final String YARN_HISTORY_SERVICE_ENABLED = AHS_PREFIX
- + "enabled";
- public static final boolean DEFAULT_YARN_HISTORY_SERVICE_ENABLED = false;
-
- /** URI for FileSystemApplicationHistoryStore */
- public static final String FS_HISTORY_STORE_URI = AHS_PREFIX + "fs-history-store.uri";
-
- /** T-file compression types used to compress history data.*/
- public static final String FS_HISTORY_STORE_COMPRESSION_TYPE = AHS_PREFIX + "fs-history-store.compression-type";
- public static final String DEFAULT_FS_HISTORY_STORE_COMPRESSION_TYPE = "none";
+ public static final String TIMELINE_SERVICE_PREFIX =
+ YARN_PREFIX + "timeline-service.";
- /** AHS store class */
- public static final String AHS_STORE = AHS_PREFIX + "store.class";
- /** host:port address for Application History Server API. */
- public static final String AHS_ADDRESS = AHS_PREFIX + "address";
- public static final int DEFAULT_AHS_PORT = 10200;
- public static final String DEFAULT_AHS_ADDRESS = "0.0.0.0:"
- + DEFAULT_AHS_PORT;
-
- /** The number of threads to handle client API requests. */
- public static final String AHS_CLIENT_THREAD_COUNT = AHS_PREFIX
- + "client.thread-count";
- public static final int DEFAULT_AHS_CLIENT_THREAD_COUNT = 10;
-
-
- /** The address of the AHS web application.*/
- public static final String AHS_WEBAPP_ADDRESS = AHS_PREFIX
- + "webapp.address";
-
- public static final int DEFAULT_AHS_WEBAPP_PORT = 8188;
- public static final String DEFAULT_AHS_WEBAPP_ADDRESS = "0.0.0.0:"
- + DEFAULT_AHS_WEBAPP_PORT;
-
- /** The https address of the AHS web application.*/
- public static final String AHS_WEBAPP_HTTPS_ADDRESS = AHS_PREFIX
- + "webapp.https.address";
+ // mark app-history related configs @Private as application history is going
+ // to be integrated into the timeline service
+ @Private
+ public static final String APPLICATION_HISTORY_PREFIX =
+ TIMELINE_SERVICE_PREFIX + "generic-application-history.";
- public static final int DEFAULT_AHS_WEBAPP_HTTPS_PORT = 8190;
- public static final String DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
- + DEFAULT_AHS_WEBAPP_HTTPS_PORT;
-
- /**The kerberos principal to be used for spnego filter for AHS.*/
- public static final String AHS_WEBAPP_SPNEGO_USER_NAME_KEY =
- AHS_PREFIX + "webapp.spnego-principal";
-
- /**The kerberos keytab to be used for spnego filter for AHS.*/
- public static final String AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
- AHS_PREFIX + "webapp.spnego-keytab-file";
+ /**
+ * The setting that controls whether application history service is
+ * enabled or not.
+ */
+ @Private
+ public static final String APPLICATION_HISTORY_ENABLED =
+ APPLICATION_HISTORY_PREFIX + "enabled";
+ @Private
+ public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false;
- ////////////////////////////////
- // ATS Configs
- ////////////////////////////////
+ /** Application history store class */
+ @Private
+ public static final String APPLICATION_HISTORY_STORE =
+ APPLICATION_HISTORY_PREFIX + "store-class";
- public static final String ATS_PREFIX = YARN_PREFIX + "ats.";
+ /** URI for FileSystemApplicationHistoryStore */
+ @Private
+ public static final String FS_APPLICATION_HISTORY_STORE_URI =
+ APPLICATION_HISTORY_PREFIX + "fs-history-store.uri";
- /** ATS store class */
- public static final String ATS_STORE = ATS_PREFIX + "store.class";
+ /** T-file compression types used to compress history data.*/
+ @Private
+ public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE =
+ APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type";
+ @Private
+ public static final String DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE =
+ "none";
- /** ATS leveldb path */
- public static final String ATS_LEVELDB_PATH_PROPERTY =
- ATS_PREFIX + "leveldb-apptimeline-store.path";
+ /** host:port address for timeline service RPC APIs. */
+ public static final String TIMELINE_SERVICE_ADDRESS =
+ TIMELINE_SERVICE_PREFIX + "address";
+ public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200;
+ public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:"
+ + DEFAULT_TIMELINE_SERVICE_PORT;
+
+ /** The number of threads to handle client RPC API requests. */
+ public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
+ TIMELINE_SERVICE_PREFIX + "handler-thread-count";
+ public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10;
+
+
+ /** The address of the timeline service web application.*/
+ public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS =
+ TIMELINE_SERVICE_PREFIX + "webapp.address";
+
+ public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188;
+ public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS =
+ "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT;
+
+ /** The https address of the timeline service web application.*/
+ public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
+ TIMELINE_SERVICE_PREFIX + "webapp.https.address";
+
+ public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190;
+ public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
+ "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+
+ /**The kerberos principal to be used for spnego filter for timeline service.*/
+ public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY =
+ TIMELINE_SERVICE_PREFIX + "webapp.spnego-principal";
+
+ /**The kerberos keytab to be used for spnego filter for timeline service.*/
+ public static final String TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+ TIMELINE_SERVICE_PREFIX + "webapp.spnego-keytab-file";
+
+ /** Timeline service store class */
+ public static final String TIMELINE_SERVICE_STORE =
+ TIMELINE_SERVICE_PREFIX + "store-class";
+
+ /** Timeline service leveldb path */
+ public static final String TIMELINE_SERVICE_LEVELDB_PATH =
+ TIMELINE_SERVICE_PREFIX + "leveldb-timeline-store.path";
////////////////////////////////
// Other Configs
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java Tue Feb 25 01:16:30 2014
@@ -24,8 +24,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -33,7 +33,7 @@ import org.apache.hadoop.yarn.exceptions
* A client library that can be used to post some information in terms of a
* number of conceptual entities.
*
- * @See ATSEntity
+ * @See Entity
*/
@Public
@Unstable
@@ -52,19 +52,19 @@ public abstract class TimelineClient ext
/**
* <p>
- * Post the information of a number of conceptual entities of an application
- * to the timeline server. It is a blocking API. The method will not return
- * until it gets the response from the timeline server.
+ * Send the information of a number of conceptual entities to the timeline
+ * server. It is a blocking API. The method will not return until it gets the
+ * response from the timeline server.
* </p>
*
* @param entities
- * the collection of {@link ATSEntity}
- * @return the error information if the post entities are not correctly stored
+ * the collection of {@link TimelineEntity}
+ * @return the error information if the sent entities are not correctly stored
* @throws IOException
* @throws YarnException
*/
@Public
- public abstract ATSPutErrors postEntities(
- ATSEntity... entities) throws IOException, YarnException;
+ public abstract TimelinePutResponse putEntities(
+ TimelineEntity... entities) throws IOException, YarnException;
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSClientImpl.java Tue Feb 25 01:16:30 2014
@@ -63,9 +63,9 @@ public class AHSClientImpl extends AHSCl
}
private static InetSocketAddress getAHSAddress(Configuration conf) {
- return conf.getSocketAddr(YarnConfiguration.AHS_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_PORT);
+ return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
}
@Override
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java Tue Feb 25 01:16:30 2014
@@ -29,9 +29,9 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -50,7 +50,7 @@ import com.sun.jersey.api.client.config.
public class TimelineClientImpl extends TimelineClient {
private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class);
- private static final String RESOURCE_URI_STR = "/ws/v1/apptimeline/";
+ private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
private static final Joiner JOINER = Joiner.on("");
private Client client;
@@ -67,21 +67,21 @@ public class TimelineClientImpl extends
if (YarnConfiguration.useHttps(conf)) {
resURI = URI
.create(JOINER.join("https://", conf.get(
- YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS),
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS),
RESOURCE_URI_STR));
} else {
resURI = URI.create(JOINER.join("http://", conf.get(
- YarnConfiguration.AHS_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS), RESOURCE_URI_STR));
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS), RESOURCE_URI_STR));
}
super.serviceInit(conf);
}
@Override
- public ATSPutErrors postEntities(
- ATSEntity... entities) throws IOException, YarnException {
- ATSEntities entitiesContainer = new ATSEntities();
+ public TimelinePutResponse putEntities(
+ TimelineEntity... entities) throws IOException, YarnException {
+ TimelineEntities entitiesContainer = new TimelineEntities();
entitiesContainer.addEntities(Arrays.asList(entities));
ClientResponse resp = doPostingEntities(entitiesContainer);
if (resp.getClientResponseStatus() != ClientResponse.Status.OK) {
@@ -95,12 +95,12 @@ public class TimelineClientImpl extends
}
throw new YarnException(msg);
}
- return resp.getEntity(ATSPutErrors.class);
+ return resp.getEntity(TimelinePutResponse.class);
}
@Private
@VisibleForTesting
- public ClientResponse doPostingEntities(ATSEntities entities) {
+ public ClientResponse doPostingEntities(TimelineEntities entities) {
WebResource webResource = client.resource(resURI);
return webResource.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Tue Feb 25 01:16:30 2014
@@ -114,8 +114,8 @@ public class YarnClientImpl extends Yarn
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
}
- if (conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED)) {
+ if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
historyServiceEnabled = true;
historyClient = AHSClientImpl.createAHSClient();
historyClient.init(getConfig());
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java Tue Feb 25 01:16:30 2014
@@ -25,10 +25,10 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import junit.framework.Assert;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntities;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSEntity;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSEvent;
-import org.apache.hadoop.yarn.api.records.apptimeline.ATSPutErrors;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -58,8 +58,8 @@ public class TestTimelineClient {
public void testPostEntities() throws Exception {
mockClientResponse(ClientResponse.Status.OK, false);
try {
- ATSPutErrors errors = client.postEntities(generateATSEntity());
- Assert.assertEquals(0, errors.getErrors().size());
+ TimelinePutResponse response = client.putEntities(generateEntity());
+ Assert.assertEquals(0, response.getErrors().size());
} catch (YarnException e) {
Assert.fail("Exception is not expected");
}
@@ -69,14 +69,14 @@ public class TestTimelineClient {
public void testPostEntitiesWithError() throws Exception {
mockClientResponse(ClientResponse.Status.OK, true);
try {
- ATSPutErrors errors = client.postEntities(generateATSEntity());
- Assert.assertEquals(1, errors.getErrors().size());
- Assert.assertEquals("test entity id", errors.getErrors().get(0)
+ TimelinePutResponse response = client.putEntities(generateEntity());
+ Assert.assertEquals(1, response.getErrors().size());
+ Assert.assertEquals("test entity id", response.getErrors().get(0)
.getEntityId());
- Assert.assertEquals("test entity type", errors.getErrors().get(0)
+ Assert.assertEquals("test entity type", response.getErrors().get(0)
.getEntityType());
- Assert.assertEquals(ATSPutErrors.ATSPutError.IO_EXCEPTION,
- errors.getErrors().get(0).getErrorCode());
+ Assert.assertEquals(TimelinePutResponse.TimelinePutError.IO_EXCEPTION,
+ response.getErrors().get(0).getErrorCode());
} catch (YarnException e) {
Assert.fail("Exception is not expected");
}
@@ -86,7 +86,7 @@ public class TestTimelineClient {
public void testPostEntitiesNoResponse() throws Exception {
mockClientResponse(ClientResponse.Status.INTERNAL_SERVER_ERROR, false);
try {
- client.postEntities(generateATSEntity());
+ client.putEntities(generateEntity());
Assert.fail("Exception is expected");
} catch (YarnException e) {
Assert.assertTrue(e.getMessage().contains(
@@ -98,27 +98,28 @@ public class TestTimelineClient {
boolean hasError) {
ClientResponse response = mock(ClientResponse.class);
doReturn(response).when(client)
- .doPostingEntities(any(ATSEntities.class));
+ .doPostingEntities(any(TimelineEntities.class));
when(response.getClientResponseStatus()).thenReturn(status);
- ATSPutErrors.ATSPutError error = new ATSPutErrors.ATSPutError();
+ TimelinePutResponse.TimelinePutError error =
+ new TimelinePutResponse.TimelinePutError();
error.setEntityId("test entity id");
error.setEntityType("test entity type");
- error.setErrorCode(ATSPutErrors.ATSPutError.IO_EXCEPTION);
- ATSPutErrors errors = new ATSPutErrors();
+ error.setErrorCode(TimelinePutResponse.TimelinePutError.IO_EXCEPTION);
+ TimelinePutResponse putResponse = new TimelinePutResponse();
if (hasError) {
- errors.addError(error);
+ putResponse.addError(error);
}
- when(response.getEntity(ATSPutErrors.class)).thenReturn(errors);
+ when(response.getEntity(TimelinePutResponse.class)).thenReturn(putResponse);
return response;
}
- private static ATSEntity generateATSEntity() {
- ATSEntity entity = new ATSEntity();
+ private static TimelineEntity generateEntity() {
+ TimelineEntity entity = new TimelineEntity();
entity.setEntityId("entity id");
entity.setEntityType("entity type");
entity.setStartTime(System.currentTimeMillis());
for (int i = 0; i < 2; ++i) {
- ATSEvent event = new ATSEvent();
+ TimelineEvent event = new TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType("test event type " + i);
event.addEventInfo("key1", "val1");
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java Tue Feb 25 01:16:30 2014
@@ -142,11 +142,11 @@ public class WebAppUtils {
public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
- return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS);
+ return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS);
} else {
- return conf.get(YarnConfiguration.AHS_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS);
+ return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Tue Feb 25 01:16:30 2014
@@ -573,30 +573,12 @@
</property>
<property>
- <description>Indicate to ResourceManager as well as clients whether
- history-service is enabled or not. If enabled, ResourceManager starts
- recording historical data that ApplicationHistory service can consume.
- Similarly, clients can redirect to the history service when applications
- finish if this is enabled.</description>
- <name>yarn.ahs.enabled</name>
- <value>false</value>
- </property>
-
- <property>
<description>Number of worker threads that write the history data.</description>
<name>yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size</name>
<value>10</value>
</property>
<property>
- <description>The implementation class of ApplicationHistoryStore, which is
- to be used by RMApplicationHistoryWriter.
- </description>
- <name>yarn.resourcemanager.history-writer.class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
- </property>
-
- <property>
<description>The class to use as the configuration provider.
If org.apache.hadoop.yarn.LocalConfigurationProvider is used,
the local configuration will be loaded.
@@ -1085,73 +1067,81 @@
<value></value>
</property>
- <!-- Application History Service's Configuration-->
+ <!-- Timeline Service's Configuration-->
<property>
- <description>The hostname of the AHS.</description>
- <name>yarn.ahs.hostname</name>
+ <description>The hostname of the timeline service web application.</description>
+ <name>yarn.timeline-service.hostname</name>
<value>0.0.0.0</value>
</property>
<property>
- <description>The http address of the AHS web application.</description>
- <name>yarn.ahs.webapp.address</name>
- <value>${yarn.ahs.hostname}:8188</value>
+ <description>This is default address for the timeline server to start the
+ RPC server.</description>
+ <name>yarn.timeline-service.address</name>
+ <value>0.0.0.0:10200</value>
</property>
<property>
- <description>The https adddress of the AHS web application.</description>
- <name>yarn.ahs.webapp.https.address</name>
- <value>${yarn.ahs.hostname}:8190</value>
+ <description>The http address of the timeline service web application.</description>
+ <name>yarn.timeline-service.webapp.address</name>
+ <value>${yarn.timeline-service.hostname}:8188</value>
</property>
<property>
- <description>URI pointing to the location of the FileSystem path where
- the history will be persisted. This must be supplied when using
- org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
- as the value for yarn.resourcemanager.history-writer.store.class</description>
- <name>yarn.ahs.fs-history-store.uri</name>
- <value>${hadoop.log.dir}/yarn/system/ahstore</value>
+ <description>The https adddress of the timeline service web application.</description>
+ <name>yarn.timeline-service.webapp.https.address</name>
+ <value>${yarn.timeline-service.hostname}:8190</value>
</property>
<property>
- <description>This is default address for the Application History server
- to start the RPC server.</description>
- <name>yarn.ahs.address</name>
- <value>0.0.0.0:10200</value>
+ <description>Store class name for timeline store</description>
+ <name>yarn.timeline-service.store-class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore</value>
</property>
<property>
- <description>CLient thread count to serve the client requests.</description>
- <name>yarn.ahs.client.thread-count</name>
- <value>10</value>
+ <description>Store file name for leveldb timeline store</description>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>${yarn.log.dir}/timeline</value>
</property>
-
+
<property>
- <description>T-file compression types used to compress history data.</description>
- <name>yarn.ahs.fs-history-store.compression-type</name>
- <value>none</value>
+ <description>Handler thread count to serve the client RPC requests.</description>
+ <name>yarn.timeline-service.handler-thread-count</name>
+ <value>10</value>
</property>
-
+
<property>
- <description> Store class name for history store, defaulting to file
- system store </description>
- <name>yarn.ahs.store.class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
+ <description>Indicate to ResourceManager as well as clients whether
+ history-service is enabled or not. If enabled, ResourceManager starts
+ recording historical data that ApplicationHistory service can consume.
+ Similarly, clients can redirect to the history service when applications
+ finish if this is enabled.</description>
+ <name>yarn.timeline-service.generic-application-history.enabled</name>
+ <value>false</value>
</property>
- <!-- Application Timeline Service's Configuration-->
-
<property>
- <description>Store class name for application timeline store</description>
- <name>yarn.ats.store.class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore</value>
+ <description>URI pointing to the location of the FileSystem path where
+ the history will be persisted. This must be supplied when using
+ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
+ as the value for yarn.timeline-service.generic-application-history.store-class</description>
+ <name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name>
+ <value>${hadoop.log.dir}/yarn/system/history</value>
</property>
<property>
- <description>Store file name for leveldb application timeline store</description>
- <name>yarn.ats.leveldb-apptimeline-store.path</name>
- <value>${yarn.log.dir}/ats</value>
+ <description>T-file compression types used to compress history data.</description>
+ <name>yarn.timeline-service.generic-application-history.fs-history-store.compression-type</name>
+ <value>none</value>
+ </property>
+
+ <property>
+ <description>Store class name for history store, defaulting to file
+ system store </description>
+ <name>yarn.timeline-service.generic-application-history.store-class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
</property>
<!-- Other configuration -->
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java Tue Feb 25 01:16:30 2014
@@ -76,19 +76,19 @@ public class ApplicationHistoryClientSer
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress address =
- conf.getSocketAddr(YarnConfiguration.AHS_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_ADDRESS,
- YarnConfiguration.DEFAULT_AHS_PORT);
+ conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
server =
rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler,
address, conf, null, conf.getInt(
- YarnConfiguration.AHS_CLIENT_THREAD_COUNT,
- YarnConfiguration.DEFAULT_AHS_CLIENT_THREAD_COUNT));
+ YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT));
server.start();
this.bindAddress =
- conf.updateConnectAddr(YarnConfiguration.AHS_ADDRESS,
+ conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
server.getListenerAddress());
LOG.info("Instantiated ApplicationHistoryClientService at "
+ this.bindAddress);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java Tue Feb 25 01:16:30 2014
@@ -79,7 +79,8 @@ public class ApplicationHistoryManagerIm
protected ApplicationHistoryStore createApplicationHistoryStore(
Configuration conf) {
return ReflectionUtils.newInstance(conf.getClass(
- YarnConfiguration.AHS_STORE, FileSystemApplicationHistoryStore.class,
+ YarnConfiguration.APPLICATION_HISTORY_STORE,
+ FileSystemApplicationHistoryStore.class,
ApplicationHistoryStore.class), conf);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Tue Feb 25 01:16:30 2014
@@ -33,8 +33,8 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.LeveldbApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
@@ -54,7 +54,7 @@ public class ApplicationHistoryServer ex
ApplicationHistoryClientService ahsClientService;
ApplicationHistoryManager historyManager;
- ApplicationTimelineStore timelineStore;
+ TimelineStore timelineStore;
private WebApp webApp;
public ApplicationHistoryServer() {
@@ -67,7 +67,7 @@ public class ApplicationHistoryServer ex
ahsClientService = createApplicationHistoryClientService(historyManager);
addService(ahsClientService);
addService((Service) historyManager);
- timelineStore = createApplicationTimelineStore(conf);
+ timelineStore = createTimelineStore(conf);
addIfService(timelineStore);
super.serviceInit(conf);
}
@@ -141,11 +141,11 @@ public class ApplicationHistoryServer ex
return new ApplicationHistoryManagerImpl();
}
- protected ApplicationTimelineStore createApplicationTimelineStore(
+ protected TimelineStore createTimelineStore(
Configuration conf) {
return ReflectionUtils.newInstance(conf.getClass(
- YarnConfiguration.ATS_STORE, LeveldbApplicationTimelineStore.class,
- ApplicationTimelineStore.class), conf);
+ YarnConfiguration.TIMELINE_SERVICE_STORE, LeveldbTimelineStore.class,
+ TimelineStore.class), conf);
}
protected void startWebApp() {
@@ -158,9 +158,9 @@ public class ApplicationHistoryServer ex
ahsClientService, "ws")
.with(getConfig())
.withHttpSpnegoPrincipalKey(
- YarnConfiguration.AHS_WEBAPP_SPNEGO_USER_NAME_KEY)
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
- YarnConfiguration.AHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
.at(bindAddress)
.start(new AHSWebApp(historyManager, timelineStore));
} catch (Exception e) {
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Tue Feb 25 01:16:30 2014
@@ -113,7 +113,7 @@ public class FileSystemApplicationHistor
@Override
public void serviceInit(Configuration conf) throws Exception {
Path fsWorkingPath =
- new Path(conf.get(YarnConfiguration.FS_HISTORY_STORE_URI));
+ new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI));
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
try {
fs = fsWorkingPath.getFileSystem(conf);
@@ -727,8 +727,8 @@ public class FileSystemApplicationHistor
fs.setPermission(historyFile, HISTORY_FILE_UMASK);
writer =
new TFile.Writer(fsdos, MIN_BLOCK_SIZE, getConfig().get(
- YarnConfiguration.FS_HISTORY_STORE_COMPRESSION_TYPE,
- YarnConfiguration.DEFAULT_FS_HISTORY_STORE_COMPRESSION_TYPE), null,
+ YarnConfiguration.FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE,
+ YarnConfiguration.DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE), null,
getConfig());
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java Tue Feb 25 01:16:30 2014
@@ -21,7 +21,7 @@ import static org.apache.hadoop.yarn.uti
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.apptimeline.ApplicationTimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
@@ -30,22 +30,22 @@ import org.apache.hadoop.yarn.webapp.Yar
public class AHSWebApp extends WebApp implements YarnWebParams {
private final ApplicationHistoryManager applicationHistoryManager;
- private final ApplicationTimelineStore applicationTimelineStore;
+ private final TimelineStore timelineStore;
public AHSWebApp(ApplicationHistoryManager applicationHistoryManager,
- ApplicationTimelineStore applicationTimelineStore) {
+ TimelineStore timelineStore) {
this.applicationHistoryManager = applicationHistoryManager;
- this.applicationTimelineStore = applicationTimelineStore;
+ this.timelineStore = timelineStore;
}
@Override
public void setup() {
bind(YarnJacksonJaxbJsonProvider.class);
bind(AHSWebServices.class);
- bind(ATSWebServices.class);
+ bind(TimelineWebServices.class);
bind(GenericExceptionHandler.class);
bind(ApplicationContext.class).toInstance(applicationHistoryManager);
- bind(ApplicationTimelineStore.class).toInstance(applicationTimelineStore);
+ bind(TimelineStore.class).toInstance(timelineStore);
route("/", AHSController.class);
route(pajoin("/apps", APP_STATE), AHSController.class);
route(pajoin("/app", APPLICATION_ID), AHSController.class, "app");
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java Tue Feb 25 01:16:30 2014
@@ -57,7 +57,7 @@ public class TestApplicationHistoryClien
public void setup() {
historyServer = new ApplicationHistoryServer();
Configuration config = new YarnConfiguration();
- config.setClass(YarnConfiguration.AHS_STORE,
+ config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
historyServer.init(config);
historyServer.start();
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java Tue Feb 25 01:16:30 2014
@@ -38,7 +38,7 @@ public class TestApplicationHistoryManag
@Before
public void setup() throws Exception {
Configuration config = new Configuration();
- config.setClass(YarnConfiguration.AHS_STORE,
+ config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
applicationHistoryManagerImpl = new ApplicationHistoryManagerImpl();
applicationHistoryManagerImpl.init(config);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Tue Feb 25 01:16:30 2014
@@ -52,7 +52,7 @@ public class TestFileSystemApplicationHi
fs.initialize(new URI("/"), conf);
fsWorkingPath = new Path("Test");
fs.delete(fsWorkingPath, true);
- conf.set(YarnConfiguration.FS_HISTORY_STORE_URI, fsWorkingPath.toString());
+ conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
store = new FileSystemApplicationHistoryStore();
store.init(conf);
store.start();
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Feb 25 01:16:30 2014
@@ -224,11 +224,16 @@ public class NodeManager extends Composi
new Thread() {
@Override
public void run() {
- LOG.info("Notifying ContainerManager to block new container-requests");
- containerManager.setBlockNewContainerRequests(true);
- LOG.info("Cleaning up running containers on resync");
- containerManager.cleanupContainersOnNMResync();
- ((NodeStatusUpdaterImpl) nodeStatusUpdater ).rebootNodeStatusUpdater();
+ try {
+ LOG.info("Notifying ContainerManager to block new container-requests");
+ containerManager.setBlockNewContainerRequests(true);
+ LOG.info("Cleaning up running containers on resync");
+ containerManager.cleanupContainersOnNMResync();
+ ((NodeStatusUpdaterImpl) nodeStatusUpdater).rebootNodeStatusUpdater();
+ } catch (YarnRuntimeException e) {
+ LOG.fatal("Error while rebooting NodeStatusUpdater.", e);
+ shutDown();
+ }
}
}.start();
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java Tue Feb 25 01:16:30 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
@@ -65,6 +66,7 @@ public class TestNodeManagerResync {
private FileContext localFS;
private CyclicBarrier syncBarrier;
private AtomicBoolean assertionFailedInThread = new AtomicBoolean(false);
+ private AtomicBoolean isNMShutdownCalled = new AtomicBoolean(false);
@Before
public void setup() throws UnsupportedFileSystemException {
@@ -137,6 +139,30 @@ public class TestNodeManagerResync {
Assert.assertFalse(assertionFailedInThread.get());
nm.stop();
}
+
+ @SuppressWarnings("unchecked")
+ @Test(timeout=10000)
+ public void testNMshutdownWhenResyncThrowException() throws IOException,
+ InterruptedException, YarnException {
+ NodeManager nm = new TestNodeManager3();
+ YarnConfiguration conf = createNMConfig();
+ nm.init(conf);
+ nm.start();
+ Assert.assertEquals(1, ((TestNodeManager3) nm).getNMRegistrationCount());
+ nm.getNMDispatcher().getEventHandler()
+ .handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
+
+ synchronized (isNMShutdownCalled) {
+ while (isNMShutdownCalled.get() == false) {
+ try {
+ isNMShutdownCalled.wait();
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+
+ Assert.assertTrue("NM shutdown not called.",isNMShutdownCalled.get());
+ }
private YarnConfiguration createNMConfig() {
YarnConfiguration conf = new YarnConfiguration();
@@ -322,4 +348,44 @@ public class TestNodeManagerResync {
}
}
}
+
+ class TestNodeManager3 extends NodeManager {
+
+ private int registrationCount = 0;
+
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ return new TestNodeStatusUpdaterImpl3(context, dispatcher, healthChecker,
+ metrics);
+ }
+
+ public int getNMRegistrationCount() {
+ return registrationCount;
+ }
+
+ @Override
+ protected void shutDown() {
+ synchronized (isNMShutdownCalled) {
+ isNMShutdownCalled.set(true);
+ isNMShutdownCalled.notify();
+ }
+ }
+
+ class TestNodeStatusUpdaterImpl3 extends MockNodeStatusUpdater {
+
+ public TestNodeStatusUpdaterImpl3(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
+ }
+
+ @Override
+ protected void registerWithRM() throws YarnException, IOException {
+ super.registerWithRM();
+ registrationCount++;
+ if (registrationCount > 1) {
+ throw new YarnRuntimeException("Registration with RM failed.");
+ }
+ }
+ }}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java Tue Feb 25 01:16:30 2014
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryWriter;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
@@ -81,8 +82,8 @@ public class RMApplicationHistoryWriter
protected synchronized void serviceInit(Configuration conf) throws Exception {
historyServiceEnabled =
- conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED);
+ conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED);
writer = createApplicationHistoryStore(conf);
addIfService(writer);
@@ -112,14 +113,15 @@ public class RMApplicationHistoryWriter
if (historyServiceEnabled) {
try {
Class<? extends ApplicationHistoryStore> storeClass =
- conf.getClass(YarnConfiguration.RM_HISTORY_WRITER_CLASS,
- NullApplicationHistoryStore.class, ApplicationHistoryStore.class);
+ conf.getClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
+ FileSystemApplicationHistoryStore.class,
+ ApplicationHistoryStore.class);
return storeClass.newInstance();
} catch (Exception e) {
String msg =
"Could not instantiate ApplicationHistoryWriter: "
- + conf.get(YarnConfiguration.RM_HISTORY_WRITER_CLASS,
- NullApplicationHistoryStore.class.getName());
+ + conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE,
+ FileSystemApplicationHistoryStore.class.getName());
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
@@ -214,21 +216,25 @@ public class RMApplicationHistoryWriter
@SuppressWarnings("unchecked")
public void applicationStarted(RMApp app) {
- dispatcher.getEventHandler().handle(
- new WritingApplicationStartEvent(app.getApplicationId(),
- ApplicationStartData.newInstance(app.getApplicationId(), app.getName(),
- app.getApplicationType(), app.getQueue(), app.getUser(),
- app.getSubmitTime(), app.getStartTime())));
+ if (historyServiceEnabled) {
+ dispatcher.getEventHandler().handle(
+ new WritingApplicationStartEvent(app.getApplicationId(),
+ ApplicationStartData.newInstance(app.getApplicationId(), app.getName(),
+ app.getApplicationType(), app.getQueue(), app.getUser(),
+ app.getSubmitTime(), app.getStartTime())));
+ }
}
@SuppressWarnings("unchecked")
public void applicationFinished(RMApp app, RMAppState finalState) {
- dispatcher.getEventHandler().handle(
- new WritingApplicationFinishEvent(app.getApplicationId(),
- ApplicationFinishData.newInstance(app.getApplicationId(),
- app.getFinishTime(), app.getDiagnostics().toString(),
- app.getFinalApplicationStatus(),
- RMServerUtils.createApplicationState(finalState))));
+ if (historyServiceEnabled) {
+ dispatcher.getEventHandler().handle(
+ new WritingApplicationFinishEvent(app.getApplicationId(),
+ ApplicationFinishData.newInstance(app.getApplicationId(),
+ app.getFinishTime(), app.getDiagnostics().toString(),
+ app.getFinalApplicationStatus(),
+ RMServerUtils.createApplicationState(finalState))));
+ }
}
@SuppressWarnings("unchecked")
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Tue Feb 25 01:16:30 2014
@@ -201,6 +201,22 @@ public class AppSchedulable extends Sche
* Assign a container to this node to facilitate {@code request}. If node does
* not have enough memory, create a reservation. This is called once we are
* sure the particular request should be facilitated by this node.
+ *
+ * @param node
+ * The node to try placing the container on.
+ * @param priority
+ * The requested priority for the container.
+ * @param request
+ * The ResourceRequest we're trying to satisfy.
+ * @param type
+ * The locality of the assignment.
+ * @param reserved
+ * Whether there's already a container reserved for this app on the node.
+ * @return
+ * If an assignment was made, returns the resources allocated to the
+ * container. If a reservation was made, returns
+ * FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was
+ * made, returns an empty resource.
*/
private Resource assignContainer(FSSchedulerNode node,
ResourceRequest request, NodeType type,
@@ -255,17 +271,6 @@ public class AppSchedulable extends Sche
LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved);
}
- if (reserved) {
- RMContainer rmContainer = node.getReservedContainer();
- Priority priority = rmContainer.getReservedPriority();
-
- // Make sure the application still needs requests at this priority
- if (app.getTotalRequiredResources(priority) == 0) {
- unreserve(priority, node);
- return Resources.none();
- }
- }
-
Collection<Priority> prioritiesToTry = (reserved) ?
Arrays.asList(node.getReservedContainer().getReservedPriority()) :
app.getPriorities();
@@ -338,7 +343,33 @@ public class AppSchedulable extends Sche
return Resources.none();
}
+ /**
+ * Called when this application already has an existing reservation on the
+ * given node. Sees whether we can turn the reservation into an allocation.
+ * Also checks whether the application needs the reservation anymore, and
+ * releases it if not.
+ *
+ * @param node
+ * Node that the application has an existing reservation on
+ */
public Resource assignReservedContainer(FSSchedulerNode node) {
+ RMContainer rmContainer = node.getReservedContainer();
+ Priority priority = rmContainer.getReservedPriority();
+
+ // Make sure the application still needs requests at this priority
+ if (app.getTotalRequiredResources(priority) == 0) {
+ unreserve(priority, node);
+ return Resources.none();
+ }
+
+ // Fail early if the reserved container won't fit.
+ // Note that we have an assumption here that there's only one container size
+ // per priority.
+ if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(),
+ node.getAvailableResource())) {
+ return Resources.none();
+ }
+
return assignContainer(node, true);
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Tue Feb 25 01:16:30 2014
@@ -1046,10 +1046,12 @@ public class FairScheduler extends Abstr
reservedAppSchedulable = null;
} else {
// Reservation exists; try to fulfill the reservation
- LOG.info("Trying to fulfill reservation for application "
- + reservedAppSchedulable.getApp().getApplicationAttemptId()
- + " on node: " + node);
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to fulfill reservation for application "
+ + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + " on node: " + node);
+ }
+
node.getReservedAppSchedulable().assignReservedContainer(node);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java?rev=1571509&r1=1571508&r2=1571509&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java Tue Feb 25 01:16:30 2014
@@ -77,7 +77,7 @@ public class TestRMApplicationHistoryWri
public void setup() {
store = new MemoryApplicationHistoryStore();
Configuration conf = new Configuration();
- conf.setBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
writer = new RMApplicationHistoryWriter() {
@Override