You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/01/28 20:58:08 UTC
svn commit: r1562198 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/resources/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-serv...
Author: vinodkv
Date: Tue Jan 28 19:58:07 2014
New Revision: 1562198
URL: http://svn.apache.org/r1562198
Log:
YARN-955. Implemented ApplicationHistoryProtocol handler. Contributed by Mayank Bansal.
svn merge --ignore-ancestry -c 1556743 ../YARN-321
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
- copied unchanged from r1556743, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
- copied unchanged from r1556743, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
- copied unchanged from r1556743, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
Removed:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/AHSClientService.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1562198&r1=1562197&r2=1562198&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Tue Jan 28 19:58:07 2014
@@ -499,6 +499,9 @@ Branch YARN-321: Generic ApplicationHist
YARN-1266. Implemented PB service and client wrappers for
ApplicationHistoryProtocol. (Mayank Bansal via vinodkv)
+ YARN-955. Implemented ApplicationHistoryProtocol handler. (Mayank Bansal via
+ vinodkv)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1562198&r1=1562197&r2=1562198&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Jan 28 19:58:07 2014
@@ -962,6 +962,17 @@ public class YarnConfiguration extends C
/** AHS store class */
public static final String AHS_STORE = AHS_PREFIX + "store.class";
+
+ /** host:port address for Application History Server API. */
+ public static final String AHS_ADDRESS = AHS_PREFIX + "address";
+ public static final int DEFAULT_AHS_PORT = 10200;
+ public static final String DEFAULT_AHS_ADDRESS = "0.0.0.0:"
+ + DEFAULT_AHS_PORT;
+
+ /** The number of threads to handle client API requests. */
+ public static final String AHS_CLIENT_THREAD_COUNT = AHS_PREFIX
+ + "client.thread-count";
+ public static final int DEFAULT_AHS_CLIENT_THREAD_COUNT = 10;
////////////////////////////////
// Other Configs
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1562198&r1=1562197&r2=1562198&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Tue Jan 28 19:58:07 2014
@@ -1074,6 +1074,19 @@
</property>
<property>
+ <description>This is default address for the Application History server
+ to start the RPC server.</description>
+ <name>yarn.ahs.address</name>
+ <value>0.0.0.0:10200</value>
+ </property>
+
+ <property>
+ <description>CLient thread count to serve the client requests.</description>
+ <name>yarn.ahs.client.thread-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
<description>T-file compression types used to compress history data.</description>
<name>yarn.ahs.fs-history-store.compression-type</name>
<value>none</value>
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1562198&r1=1562197&r2=1562198&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Tue Jan 28 19:58:07 2014
@@ -1,25 +1,38 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.yarn.server.applicationhistoryservice;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* History server that keeps track of all types of history in the cluster.
@@ -27,16 +40,82 @@ import org.apache.hadoop.service.Composi
*/
public class ApplicationHistoryServer extends CompositeService {
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private static final Log LOG = LogFactory
+ .getLog(ApplicationHistoryServer.class);
+
+ ApplicationHistoryClientService ahsClientService;
+ ApplicationHistoryManager historyManager;
+
public ApplicationHistoryServer() {
super(ApplicationHistoryServer.class.getName());
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- AHSClientService ahsClientService = new AHSClientService();
+ historyManager = createApplicationHistory();
+ ahsClientService = createApplicationHistoryClientService(historyManager);
addService(ahsClientService);
+ addService((Service) historyManager);
AHSWebServer webServer = new AHSWebServer();
addService(webServer);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ DefaultMetricsSystem.initialize("ApplicationHistoryServer");
+ JvmMetrics.initSingleton("ApplicationHistoryServer", null);
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ DefaultMetricsSystem.shutdown();
+ super.serviceStop();
+ }
+
+ @Private
+ @VisibleForTesting
+ public ApplicationHistoryClientService getClientService() {
+ return this.ahsClientService;
+ }
+
+ protected ApplicationHistoryClientService createApplicationHistoryClientService(
+ ApplicationHistoryManager historyManager) {
+ return new ApplicationHistoryClientService(historyManager);
+ }
+
+ protected ApplicationHistoryManager createApplicationHistory() {
+ return new ApplicationHistoryManagerImpl();
+ }
+
+ protected ApplicationHistoryManager getApplicationHistory() {
+ return this.historyManager;
+ }
+
+ static ApplicationHistoryServer launchAppHistoryServer(String[] args) {
+ Thread
+ .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(ApplicationHistoryServer.class, args,
+ LOG);
+ ApplicationHistoryServer appHistoryServer = null;
+ try {
+ appHistoryServer = new ApplicationHistoryServer();
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(appHistoryServer),
+ SHUTDOWN_HOOK_PRIORITY);
+ YarnConfiguration conf = new YarnConfiguration();
+ appHistoryServer.init(conf);
+ appHistoryServer.start();
+ } catch (Throwable t) {
+ LOG.fatal("Error starting ApplicationHistoryServer", t);
+ ExitUtil.terminate(-1, "Error starting ApplicationHistoryServer");
+ }
+ return appHistoryServer;
+ }
+
+ public static void main(String[] args) {
+ launchAppHistoryServer(args);
}
}