You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/27 09:24:20 UTC
[13/22] git commit: SLIDER-116. Log handling for long-lived
applications (Gour Saha via smohanty)
SLIDER-116. Log handling for long-lived applications (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a34837ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a34837ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a34837ce
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: a34837ce1d0431c960f956251eb57e23054253d1
Parents: 314c3cc
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 24 13:50:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 24 13:51:00 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/resources.json | 3 +
.../accumulo/src/test/resources/resources.json | 3 +
app-packages/hbase-win/resources.json | 3 +
.../hbase/src/test/resources/resources.json | 3 +
app-packages/storm/resources.json | 3 +
.../org/apache/slider/api/ResourceKeys.java | 3 +
.../org/apache/slider/client/SliderClient.java | 2 +
.../slider/core/launch/AbstractLauncher.java | 51 +++++++++
.../slider/core/launch/AppMasterLauncher.java | 4 +-
.../core/launch/TestAppMasterLauncher.java | 112 +++++++++++++++++++
10 files changed, 186 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/resources.json b/app-packages/accumulo/resources.json
index f876901..f9cae60 100644
--- a/app-packages/accumulo/resources.json
+++ b/app-packages/accumulo/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/src/test/resources/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/resources.json b/app-packages/accumulo/src/test/resources/resources.json
index 2711ba3..cfd9b3f 100644
--- a/app-packages/accumulo/src/test/resources/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase-win/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/resources.json b/app-packages/hbase-win/resources.json
index e0ff26f..eb38726 100644
--- a/app-packages/hbase-win/resources.json
+++ b/app-packages/hbase-win/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase/src/test/resources/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/src/test/resources/resources.json b/app-packages/hbase/src/test/resources/resources.json
index e0ff26f..eb38726 100644
--- a/app-packages/hbase/src/test/resources/resources.json
+++ b/app-packages/hbase/src/test/resources/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index 2805a3b..acecd91 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -3,6 +3,9 @@
"metadata" : {
},
"global" : {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index b542f1a..56961c9 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -118,4 +118,7 @@ public interface ResourceKeys {
*/
int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5;
+ String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
+ String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
+ String YARN_LOG_INTERVAL = "yarn.log.interval";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 129f851..afbb4a8 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1032,6 +1032,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
MapOperations sliderAMResourceComponent =
resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
+ MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions();
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
@@ -1043,6 +1044,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
yarnClient,
clusterSecure,
sliderAMResourceComponent,
+ resourceGlobalOptions,
applicationTags);
ApplicationId appId = amLauncher.getApplicationId();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 366fc8b..d3bb34c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -19,6 +19,8 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -28,6 +30,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.api.ResourceKeys;
@@ -70,6 +73,7 @@ public abstract class AbstractLauncher extends Configured {
new HashMap<String, ByteBuffer>();
// security
protected final Credentials credentials = new Credentials();
+ protected LogAggregationContext logAggregationContext;
protected AbstractLauncher(Configuration conf,
@@ -258,6 +262,53 @@ public abstract class AbstractLauncher extends Configured {
}
}
+ public void extractLogAggregationContext(Map<String, String> map) {
+ if (map != null) {
+ String logPatternSepStr = "\\|";
+ String logPatternJoinStr = "|";
+ MapOperations options = new MapOperations("", map);
+
+ List<String> logIncludePatterns = new ArrayList<String>();
+ String includePatternExpression = options.getOption(
+ ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
+ if (!includePatternExpression.isEmpty()) {
+ String[] includePatterns = includePatternExpression
+ .split(logPatternSepStr);
+ for (String includePattern : includePatterns) {
+ String trimmedIncludePattern = includePattern.trim();
+ if (!trimmedIncludePattern.isEmpty()) {
+ logIncludePatterns.add(trimmedIncludePattern);
+ }
+ }
+ }
+ String logIncludePattern = StringUtils.join(logIncludePatterns,
+ logPatternJoinStr);
+ log.info("Log include patterns: {}", logIncludePattern);
+
+ List<String> logExcludePatterns = new ArrayList<String>();
+ String excludePatternExpression = options.getOption(
+ ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
+ if (!excludePatternExpression.isEmpty()) {
+ String[] excludePatterns = excludePatternExpression
+ .split(logPatternSepStr);
+ for (String excludePattern : excludePatterns) {
+ String trimmedExcludePattern = excludePattern.trim();
+ if (!trimmedExcludePattern.isEmpty()) {
+ logExcludePatterns.add(trimmedExcludePattern);
+ }
+ }
+ }
+ String logExcludePattern = StringUtils.join(logExcludePatterns,
+ logPatternJoinStr);
+ log.info("Log exclude patterns: {}", logExcludePattern);
+
+ long logInterval = options
+ .getOptionInt(ResourceKeys.YARN_LOG_INTERVAL, 0);
+ log.info("Log interval: {}", logInterval);
+ logAggregationContext = LogAggregationContext.newInstance(
+ logIncludePattern, logExcludePattern, logInterval);
+ }
+ }
/**
* Utility method to set up the classpath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index c9ac20c..f1eeabd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -77,6 +77,7 @@ public class AppMasterLauncher extends AbstractLauncher {
SliderYarnClientImpl yarnClient,
boolean secureCluster,
Map<String, String> options,
+ Map<String, String> resourceGlobalOptions,
Set<String> applicationTags
) throws IOException, YarnException {
super(conf, fs);
@@ -96,7 +97,7 @@ public class AppMasterLauncher extends AbstractLauncher {
submissionContext.setApplicationTags(applicationTags);
}
extractResourceRequirements(resource, options);
-
+ extractLogAggregationContext(resourceGlobalOptions);
}
public void setMaxAppAttempts(int maxAppAttempts) {
@@ -170,6 +171,7 @@ public class AppMasterLauncher extends AbstractLauncher {
//container requirements
submissionContext.setResource(resource);
+ submissionContext.setLogAggregationContext(logAggregationContext);
if (keepContainersOverRestarts) {
log.debug("Requesting cluster stays running over AM failure");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
new file mode 100644
index 0000000..ca1e9ec
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderKeys;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAppMasterLauncher {
+ SliderYarnClientImpl mockYarnClient;
+ YarnClientApplication yarnClientApp;
+ ApplicationSubmissionContext appSubmissionContext;
+ Set<String> tags = Collections.emptySet();
+
+ @Before
+ public void initialize() throws Exception {
+ mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
+ yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
+ appSubmissionContext = EasyMock
+ .createNiceMock(ApplicationSubmissionContext.class);
+ EasyMock.expect(yarnClientApp.getApplicationSubmissionContext())
+ .andReturn(appSubmissionContext).once();
+ EasyMock.expect(mockYarnClient.createApplication())
+ .andReturn(yarnClientApp).once();
+ }
+
+ @Test
+ public void testExtractLogAggregationContext() throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS,
+ " | slider*.txt |agent.out| |");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+ options.put(ResourceKeys.YARN_LOG_INTERVAL, "30");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
+ SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
+ tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "slider*.txt|agent.out";
+ Assert.assertEquals(expectedInclude,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+
+ String expectedExclude = "command*.json|agent.log*";
+ Assert.assertEquals(expectedExclude,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+
+ Assert.assertEquals(30,
+ appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+ @Test
+ public void testExtractLogAggregationContextEmptyIncludePattern()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+ options.put(ResourceKeys.YARN_LOG_INTERVAL, "600");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
+ SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
+ tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "";
+ Assert.assertEquals(expectedInclude,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+
+ String expectedExclude = "command*.json|agent.log*";
+ Assert.assertEquals(expectedExclude,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+
+ Assert.assertEquals(600,
+ appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+}