You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by tg...@apache.org on 2012/08/17 22:32:42 UTC
svn commit: r1374427 - in
/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/
h...
Author: tgraves
Date: Fri Aug 17 20:32:42 2012
New Revision: 1374427
URL: http://svn.apache.org/viewvc?rev=1374427&view=rev
Log:
merge -r 1374423:1374424 from trunk. FIXES: YARN-25
Added:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
- copied unchanged from r1374424, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java
- copied unchanged from r1374424, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/CHANGES.txt?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/CHANGES.txt Fri Aug 17 20:32:42 2012
@@ -47,3 +47,5 @@ Release 0.23.3 - Unreleased
YARN-14. Symlinks to peer distributed cache files no longer work
(Jason Lowe via bobby)
+ YARN-25. remove old aggregated logs (Robert Evans via tgraves)
+
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Aug 17 20:32:42 2012
@@ -353,6 +353,14 @@ public class YarnConfiguration extends C
+ "log-aggregation-enable";
public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false;
+ /**
+ * How long to wait before deleting aggregated logs, -1 disables.
+ * Be careful set this too small and you will spam the name node.
+ */
+ public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
+ + "log-aggregation.retain-seconds";
+ public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
+
/**
* Number of seconds to retain logs on the NodeManager. Only applicable if Log
* aggregation is disabled
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java Fri Aug 17 20:32:42 2012
@@ -1,3 +1,21 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
package org.apache.hadoop.yarn.webapp.log;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER;
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java Fri Aug 17 20:32:42 2012
@@ -1,3 +1,21 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
package org.apache.hadoop.yarn.webapp.log;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID;
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Aug 17 20:32:42 2012
@@ -367,6 +367,13 @@
<name>yarn.log-aggregation-enable</name>
<value>false</value>
</property>
+
+ <property>
+ <description>How long to keep aggregation logs before deleting them. -1 disables.
+ Be careful set this too small and you will spam the name node.</description>
+ <name>yarn.log-aggregation.retain-seconds</name>
+ <value>-1</value>
+ </property>
<property>
<description>Time in seconds to retain user logs. Only applicable if
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Fri Aug 17 20:32:42 2012
@@ -70,7 +70,7 @@ public class LogAggregationService exten
/*
* Expected deployment TLD will be 1777, owner=<NMOwner>, group=<NMGroup -
- * Group to which NMOwner belongs> App dirs will be created as 750,
+ * Group to which NMOwner belongs> App dirs will be created as 770,
* owner=<AppOwner>, group=<NMGroup>: so that the owner and <NMOwner> can
* access / modify the files.
* <NMGroup> should obviously be a limited access group.
@@ -85,7 +85,7 @@ public class LogAggregationService exten
* Permissions for the Application directory.
*/
private static final FsPermission APP_DIR_PERMISSIONS = FsPermission
- .createImmutable((short) 0750);
+ .createImmutable((short) 0770);
private final Context context;
private final DeletionService deletionService;
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm?rev=1374427&r1=1374426&r2=1374427&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm (original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm Fri Aug 17 20:32:42 2012
@@ -314,6 +314,19 @@ Hadoop MapReduce Next Generation - Clust
| | | Shuffle service that needs to be set for Map Reduce applications. |
*-------------------------+-------------------------+------------------------+
+ * Configurations for History Server (Needs to be moved elsewhere):
+
+*-------------------------+-------------------------+------------------------+
+|| Parameter || Value || Notes |
+*-------------------------+-------------------------+------------------------+
+| <<<yarn.log-aggregation.retain-seconds>>> | | |
+| | <-1> | |
+| | | How long to keep aggregation logs before deleting them. -1 disables. |
+| | | Be careful, set this too small and you will spam the name node. |
+*-------------------------+-------------------------+------------------------+
+
+
+
* <<<conf/mapred-site.xml>>>
* Configurations for MapReduce Applications: