You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by go...@apache.org on 2015/05/20 07:38:20 UTC
incubator-slider git commit: SLIDER-810 YARN config changes to enable
partial logs upload for long running services (fix include pattern and tests)
Repository: incubator-slider
Updated Branches:
refs/heads/develop e94aa2696 -> 333b63572
SLIDER-810 YARN config changes to enable partial logs upload for long running services (fix include pattern and tests)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/333b6357
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/333b6357
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/333b6357
Branch: refs/heads/develop
Commit: 333b63572fb50576444a3d09287c4af37096faf8
Parents: e94aa26
Author: Gour Saha <go...@apache.org>
Authored: Tue May 19 22:38:12 2015 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Tue May 19 22:38:12 2015 -0700
----------------------------------------------------------------------
.../slider/core/launch/AbstractLauncher.java | 10 +--
.../core/launch/TestAppMasterLauncher.java | 80 +++++++++++++++-----
2 files changed, 66 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/333b6357/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 3e9b50b..93aff08 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -343,14 +343,12 @@ public abstract class AbstractLauncher extends Configured {
// patterns are left empty by the app owner in resources file
if (StringUtils.isEmpty(logIncludePattern)
&& StringUtils.isEmpty(logExcludePattern)) {
- logIncludePattern = "*";
+ logIncludePattern = ".*";
logExcludePattern = "";
- }
- if (StringUtils.isEmpty(logIncludePattern)
+ } else if (StringUtils.isEmpty(logIncludePattern)
&& StringUtils.isNotEmpty(logExcludePattern)) {
- logIncludePattern = "*";
- }
- if (StringUtils.isNotEmpty(logIncludePattern)
+ logIncludePattern = ".*";
+ } else if (StringUtils.isNotEmpty(logIncludePattern)
&& StringUtils.isEmpty(logExcludePattern)) {
logExcludePattern = "";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/333b6357/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
index a71cee0..60af770 100644
--- a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -18,12 +18,14 @@
package org.apache.slider.core.launch;
+import java.lang.reflect.Method;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.client.SliderYarnClientImpl;
@@ -38,6 +40,10 @@ public class TestAppMasterLauncher {
YarnClientApplication yarnClientApp;
ApplicationSubmissionContext appSubmissionContext;
Set<String> tags = Collections.emptySet();
+ AppMasterLauncher appMasterLauncher = null;
+ boolean isOldApi = true;
+ Method rolledLogsIncludeMethod = null;
+ Method rolledLogsExcludeMethod = null;
@Before
public void initialize() throws Exception {
@@ -49,6 +55,18 @@ public class TestAppMasterLauncher {
.andReturn(appSubmissionContext).once();
EasyMock.expect(mockYarnClient.createApplication())
.andReturn(yarnClientApp).once();
+
+ try {
+ LogAggregationContext.class.getMethod("newInstance", String.class,
+ String.class, String.class, String.class);
+ isOldApi = false;
+ rolledLogsIncludeMethod = LogAggregationContext.class
+ .getMethod("getRolledLogsIncludePattern");
+ rolledLogsExcludeMethod = LogAggregationContext.class
+ .getMethod("getRolledLogsExcludePattern");
+ } catch (Exception e) {
+ isOldApi = true;
+ }
}
/**
@@ -69,45 +87,71 @@ public class TestAppMasterLauncher {
"command*.json| agent.log* | ");
EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
- AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
- SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
- tags);
+ appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags);
// Verify the include/exclude patterns
String expectedInclude = "slider*.txt|agent.out";
- Assert.assertEquals(expectedInclude,
- appMasterLauncher.logAggregationContext.getIncludePattern());
-
String expectedExclude = "command*.json|agent.log*";
- Assert.assertEquals(expectedExclude,
- appMasterLauncher.logAggregationContext.getExcludePattern());
+ assertPatterns(expectedInclude, expectedExclude);
EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+
}
@Test
public void testExtractLogAggregationContextEmptyIncludePattern()
- throws Exception {
+ throws Exception {
Map<String, String> options = new HashMap<String, String>();
options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
"command*.json| agent.log* | ");
EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
- AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
- SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
- tags);
+ appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags);
// Verify the include/exclude patterns
- String expectedInclude = "";
- Assert.assertEquals(expectedInclude,
- appMasterLauncher.logAggregationContext.getIncludePattern());
-
+ String expectedInclude = isOldApi ? "" : ".*";
String expectedExclude = "command*.json|agent.log*";
- Assert.assertEquals(expectedExclude,
- appMasterLauncher.logAggregationContext.getExcludePattern());
+ assertPatterns(expectedInclude, expectedExclude);
EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
}
+ @Test
+ public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " ");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = isOldApi ? "" : ".*";
+ String expectedExclude = "";
+ assertPatterns(expectedInclude, expectedExclude);
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+ private void assertPatterns(String expectedIncludePattern,
+ String expectedExcludePattern) throws Exception {
+ if (isOldApi) {
+ Assert.assertEquals(expectedIncludePattern,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+ Assert.assertEquals(expectedExcludePattern,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+ } else {
+ Assert.assertEquals(expectedIncludePattern,
+ (String) rolledLogsIncludeMethod
+ .invoke(appMasterLauncher.logAggregationContext));
+ Assert.assertEquals(expectedExcludePattern,
+ (String) rolledLogsExcludeMethod
+ .invoke(appMasterLauncher.logAggregationContext));
+ }
+ }
}