You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/11/02 15:25:13 UTC
[01/50] git commit: Fix bad reference to rat-plugin
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-531-registry-enhancements 64836e835 -> b51f58112 (forced update)
Fix bad reference to rat-plugin
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/196466d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/196466d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/196466d0
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 196466d0e8875f8d9f91f2225ba0dcbfe7e64b00
Parents: 112de0e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Oct 28 19:28:38 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Oct 28 19:28:38 2014 -0700
----------------------------------------------------------------------
app-packages/command-logger/slider-pkg/pom.xml | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/196466d0/app-packages/command-logger/slider-pkg/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml
index 4550ae1..49128b4 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -112,7 +112,7 @@
<plugins>
<plugin>
- <groupId>rat</groupId>
+ <groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>${apache-rat-plugin.version}</version>
<executions>
@@ -123,6 +123,11 @@
</goals>
</execution>
</executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ </excludes>
+ </configuration>
</plugin>
</plugins>
[29/50] git commit: SLIDER-570 monkey can simulate launch failure,
ongoing work on tests
Posted by st...@apache.org.
SLIDER-570 monkey can simulate launch failure, ongoing work on tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cd37f57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cd37f57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cd37f57
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 1cd37f57053933065baf864d109fba1897ee8b95
Parents: f63ac6c
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 18:32:59 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:50 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/InternalKeys.java | 16 ++
.../org/apache/slider/client/SliderClient.java | 20 +--
.../slider/common/params/ActionDestroyArgs.java | 1 +
.../launch/SerializedApplicationReport.java | 5 -
.../server/appmaster/SliderAppMaster.java | 20 ++-
.../server/appmaster/monkey/ChaosEntry.java | 9 +-
.../appmaster/monkey/ChaosMonkeyService.java | 27 ++--
.../model/monkey/TestMockMonkey.groovy | 9 +-
.../funtest/framework/CommandTestBase.groovy | 55 +++++--
.../slider/funtest/framework/SliderShell.groovy | 1 -
.../funtest/lifecycle/AMFailuresIT.groovy | 4 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 3 +-
.../lifecycle/AgentLaunchFailureIT.groovy | 12 +-
.../lifecycle/AppsThroughAgentDemo.groovy | 160 +++++++++++++++++++
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 8 +-
15 files changed, 293 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index 074644d..b360fbe 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -150,6 +150,12 @@ public interface InternalKeys {
int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
/**
+ * Probability of a monkey check killing the AM: {@value}
+ */
+ String CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE =
+ CHAOS_MONKEY_PROBABILITY + ".amlaunchfailure";
+
+ /**
* Probability of a monkey check killing a container: {@value}
*/
@@ -162,4 +168,14 @@ public interface InternalKeys {
int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
+ /**
+ * 1% of chaos
+ */
+ int PROBABILITY_PERCENT_1 = 100;
+
+ /**
+ * 100% for chaos values
+ */
+ int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index b2e343d..0e87906 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -347,7 +347,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} catch (PathNotFoundException nfe) {
throw new NotFoundException(nfe, nfe.toString());
}
-
}
/**
@@ -529,7 +528,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderUtils.validateClusterName(clustername);
//no=op, it is now mandatory.
verifyBindingsDefined();
- verifyNoLiveClusters(clustername);
+ verifyNoLiveClusters(clustername, "Destroy");
// create the directory path
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
@@ -772,7 +771,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderUtils.validateClusterName(clustername);
verifyBindingsDefined();
if (!liveClusterAllowed) {
- verifyNoLiveClusters(clustername);
+ verifyNoLiveClusters(clustername, "Create");
}
Configuration conf = getConfig();
@@ -1100,7 +1099,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
deployedClusterName = clustername;
SliderUtils.validateClusterName(clustername);
- verifyNoLiveClusters(clustername);
+ verifyNoLiveClusters(clustername, "Launch");
Configuration config = getConfig();
lookupZKQuorum();
boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
@@ -1534,17 +1533,21 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* verify that a live cluster isn't there
* @param clustername cluster name
+ * @param action
* @throws SliderException with exit code EXIT_CLUSTER_LIVE
* if a cluster of that name is either live or starting up.
*/
- public void verifyNoLiveClusters(String clustername) throws
+ public void verifyNoLiveClusters(String clustername, String action) throws
IOException,
YarnException {
List<ApplicationReport> existing = findAllLiveInstances(clustername);
if (!existing.isEmpty()) {
throw new SliderException(EXIT_APPLICATION_IN_USE,
- clustername + ": " + E_CLUSTER_RUNNING + " :" +
+ action +" failed for "
+ + clustername
+ + ": "
+ + E_CLUSTER_RUNNING + " :" +
existing.get(0));
}
}
@@ -2168,10 +2171,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
SliderUtils.validateClusterName(clustername);
- // see if it is actually running and bail out;
verifyBindingsDefined();
- verifyNoLiveClusters(clustername);
-
+ // see if it is actually running and bail out;
+ verifyNoLiveClusters(clustername, "Start");
//start the cluster
return startCluster(clustername, thaw);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
index d4acea6..1203d28 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
@@ -24,6 +24,7 @@ import com.beust.jcommander.Parameters;
commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
public class ActionDestroyArgs extends AbstractActionArgs {
+
@Override
public String getActionName() {
return SliderActions.ACTION_DESTROY;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
index e3d77d0..dfa037d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
@@ -18,12 +18,7 @@
package org.apache.slider.core.launch;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 855ecd7..5e2ba11 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -192,7 +192,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceStateChangeListener,
RoleKeys,
ProviderCompleted {
-
+
protected static final Logger log =
LoggerFactory.getLogger(SliderAppMaster.class);
@@ -220,7 +220,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Singleton of metrics registry
*/
public static final MetricRegistry metrics = new MetricRegistry();
-
+ public static final String E_TRIGGERED_LAUNCH_FAILURE =
+ "Chaos monkey triggered launch failure";
+
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
@@ -2191,6 +2193,20 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
initAndAddService(monkey);
// configure the targets
+
+ // launch failure: special case with explicit failure triggered now
+ int amLaunchFailProbability = internals.getOptionInt(
+ InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
+ 0);
+ if (amLaunchFailProbability> 0 && monkey.chaosCheck(amLaunchFailProbability)) {
+ // trigger a failure
+ ActionStopSlider stop = new ActionStopSlider("stop",
+ 0, TimeUnit.SECONDS,
+ LauncherExitCodes.EXIT_FALSE,
+ FinalApplicationStatus.FAILED,
+ E_TRIGGERED_LAUNCH_FAILURE);
+ }
+
int amKillProbability = internals.getOptionInt(
InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 2869fe9..87a0aaa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -22,6 +22,7 @@ import com.codahale.metrics.Counter;
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.slider.api.InternalKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,16 +43,16 @@ public class ChaosEntry {
/**
* Constructor -includes validation of all arguments
- * @param name
- * @param target
- * @param probability
+ * @param name entry name
+ * @param target target
+ * @param probability probability of occurring
*/
public ChaosEntry(String name, ChaosTarget target, long probability,
MetricRegistry metrics) {
Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
Preconditions.checkArgument(target != null, "null target");
Preconditions.checkArgument(probability > 0, "negative probability");
- Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
+ Preconditions.checkArgument(probability <= InternalKeys.PROBABILITY_PERCENT_100,
"probability over 100%: "+ probability);
this.name = name;
this.target = target;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index 80f981c..27219e4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -20,6 +20,7 @@ package org.apache.slider.server.appmaster.monkey;
import com.codahale.metrics.MetricRegistry;
import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.slf4j.Logger;
@@ -36,13 +37,7 @@ import java.util.concurrent.TimeUnit;
public class ChaosMonkeyService extends AbstractService {
protected static final Logger log =
LoggerFactory.getLogger(ChaosMonkeyService.class);
- public static final int PERCENT_1 = 100;
- public static final double PERCENT_1D = 100.0;
-
- /**
- * the percentage value as multiplied up
- */
- public static final int PERCENT_100 = 100 * PERCENT_1;
+
private final MetricRegistry metrics;
private final QueueAccess queues;
private final Random random = new Random();
@@ -65,7 +60,7 @@ public class ChaosMonkeyService extends AbstractService {
public synchronized void addTarget(String name,
ChaosTarget target, long probability) {
if (probability > 0) {
- log.info("Adding {} with probability {}", name, probability / PERCENT_1);
+ log.info("Adding {} with probability {}", name, probability / InternalKeys.PROBABILITY_PERCENT_1);
chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
} else {
log.debug("Action {} not enabled", name);
@@ -85,11 +80,25 @@ public class ChaosMonkeyService extends AbstractService {
*/
public void play() {
for (ChaosEntry chaosEntry : chaosEntries) {
- long p = random.nextInt(PERCENT_100);
+ long p = randomPercentage();
chaosEntry.maybeInvokeChaos(p);
}
}
+ public int randomPercentage() {
+ return random.nextInt(InternalKeys.PROBABILITY_PERCENT_100);
+ }
+
+ /**
+ * Check for callers to see if chaos should be triggered; shares the
+ * same random number source as the rest of the monkey entries
+ * @param probability probability
+ * @return true if the action should happen
+ */
+ public boolean chaosCheck(long probability) {
+ return randomPercentage() < probability;
+ }
+
/**
* Schedule the monkey
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
index d31c9f6..e4a42fc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.server.appmaster.model.monkey
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.InternalKeys
import org.apache.slider.server.appmaster.actions.ActionHalt
import org.apache.slider.server.appmaster.actions.ActionKillContainer
import org.apache.slider.server.appmaster.actions.QueueService
@@ -66,7 +67,7 @@ class TestMockMonkey extends BaseMockAppStateTest {
@Test
public void testMonkeyPlay() throws Throwable {
ChaosCounter counter = new ChaosCounter()
- monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
+ monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100)
assert 1 == monkey.targetCount;
monkey.play()
assert counter.count == 1
@@ -76,7 +77,7 @@ class TestMockMonkey extends BaseMockAppStateTest {
public void testMonkeySchedule() throws Throwable {
ChaosCounter counter = new ChaosCounter()
assert 0 == monkey.targetCount;
- monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
+ monkey.addTarget("target", counter, InternalKeys.PROBABILITY_PERCENT_100)
assert 1 == monkey.targetCount;
assert monkey.schedule(0, 1, TimeUnit.SECONDS)
assert 1 == queues.scheduledActions.size()
@@ -105,8 +106,8 @@ class TestMockMonkey extends BaseMockAppStateTest {
public void testMonkeyPlaySometimes() throws Throwable {
ChaosCounter counter = new ChaosCounter()
ChaosCounter counter2 = new ChaosCounter()
- monkey.addTarget("target1", counter, ChaosMonkeyService.PERCENT_1 * 50)
- monkey.addTarget("target2", counter2, ChaosMonkeyService.PERCENT_1 * 25)
+ monkey.addTarget("target1", counter, InternalKeys.PROBABILITY_PERCENT_1 * 50)
+ monkey.addTarget("target2", counter2, InternalKeys.PROBABILITY_PERCENT_1 * 25)
for (int i = 0; i < 100; i++) {
monkey.play()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 1c65394..7c479af 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -457,6 +457,7 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param cluster
*/
static void setupCluster(String cluster) {
+ describe "setting up $cluster"
ensureClusterDestroyed(cluster)
}
@@ -655,7 +656,6 @@ abstract class CommandTestBase extends SliderTestUtils {
SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
commands.addAll(extraArgs)
SliderShell shell = new SliderShell(commands)
- shell.execute()
if (0 != shell.execute()) {
// app has failed.
@@ -678,7 +678,7 @@ abstract class CommandTestBase extends SliderTestUtils {
return shell
}
- public File createAppReportFile() {
+ public static File createAppReportFile() {
File reportFile = File.createTempFile(
"launch",
".json",
@@ -693,7 +693,7 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param option option to probe and use
* @return the (possibly extended) list
*/
- public List<String> maybeAddCommandOption(
+ public static List<String> maybeAddCommandOption(
List<String> args, List<String> commands, String option) {
if ( SliderUtils.isSet(option)) {
args.addAll(commands)
@@ -702,7 +702,7 @@ abstract class CommandTestBase extends SliderTestUtils {
return args
}
- public SerializedApplicationReport maybeLoadAppReport(File reportFile) {
+ public static SerializedApplicationReport maybeLoadAppReport(File reportFile) {
if (reportFile.exists() && reportFile.length()> 0) {
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
@@ -711,7 +711,7 @@ abstract class CommandTestBase extends SliderTestUtils {
return null;
}
- public SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
+ public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
def report = maybeLoadAppReport(launchReport)
if (report) {
return lookupApplication(report.applicationId)
@@ -725,7 +725,7 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param id application ID
* @return an application report or null
*/
- public SerializedApplicationReport lookupApplication(String id) {
+ public static SerializedApplicationReport lookupApplication(String id) {
File reportFile = createAppReportFile();
try {
def shell = lookup(id, reportFile)
@@ -787,6 +787,7 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
+
protected void ensureApplicationIsUp(String application) {
repeatUntilTrue(this.&isApplicationRunning,
30,
@@ -816,26 +817,58 @@ abstract class CommandTestBase extends SliderTestUtils {
protected boolean isApplicationRunning(Map<String, String> args) {
String applicationName = args['application'];
- return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
+ return isApplicationUp(applicationName);
}
protected boolean isApplicationUp(String applicationName) {
- return isApplicationInState(YarnApplicationState.RUNNING, applicationName);
+ return isApplicationInState(
+ applicationName,
+ YarnApplicationState.RUNNING
+ );
}
+ protected void ensureYarnApplicationIsUp(String application) {
+ repeatUntilTrue(this.&isApplicationRunning,
+ 30,
+ SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
+ DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
+ [application: application],
+ true,
+ 'Application did not start, failing test.') {
+ describe "final state of app that tests say is not up"
+ exists(application, true).dumpOutput()
+ }
+ }
+
/**
- *
+ * is an application in a desired yarn state
* @param yarnState
* @param applicationName
* @return
*/
- public static boolean isApplicationInState(YarnApplicationState yarnState, String applicationName) {
+ public static boolean isApplicationInState(
+ String applicationName,
+ YarnApplicationState yarnState) {
SliderShell shell = slider(
[ACTION_EXISTS, applicationName, ARG_STATE, yarnState.toString()])
-
return shell.ret == 0
}
+
+ /**
+ * is a yarn application in a desired yarn state
+ * @param yarnState
+ * @param applicationName
+ * @return
+ */
+ public static boolean isYarnApplicationInState(
+ String applicationId,
+ YarnApplicationState yarnState) {
+ def sar = lookupApplication(applicationId)
+ assert sar != null;
+ return yarnState.toString() == sar.state
+ }
+
/**
* Repeat a probe until it succeeds, if it does not execute a failure
* closure then raise an exception with the supplied message
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 31830d9..ae40d6a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -247,7 +247,6 @@ class SliderShell extends Shell {
Shell exec(Object... args) {
Process proc = "$shell".execute()
script = args.join("\n")
- LOG.debug("${shell} << __EOT__\n${script}\n__EOT__");
ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
proc.consumeProcessOutput(baosOut, baosErr)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 10a3dfe..750a3d4 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -92,8 +92,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
killAMUsingVagrantShell()
// Check that the application is not running (and is in ACCEPTED state)
- assert isApplicationInState(YarnApplicationState.ACCEPTED,
- APPLICATION_NAME),
+ assert isApplicationInState(APPLICATION_NAME, YarnApplicationState.ACCEPTED
+ ),
'App should be in ACCEPTED state (since AM got killed)'
log.info("After AM KILL: application {} is in ACCEPTED state", APPLICATION_NAME)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 8b123c3..3783a37 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -48,7 +48,6 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
@Before
public void prepareCluster() {
setupCluster(CLUSTER)
- describe("Create a 0-role cluster, so testing AM start/stop")
}
@After
@@ -62,7 +61,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
describe "Walk a 0-role cluster through its lifecycle"
// sanity check to verify the config is correct
- assert clusterFS.uri.scheme!="file"
+ assert clusterFS.uri.scheme != "file"
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
index 1a0d2c3..4dcbed0 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -20,7 +20,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.api.InternalKeys
+import static org.apache.slider.api.InternalKeys.*
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -41,7 +41,6 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
-
@Before
public void prepareCluster() {
setupCluster(CLUSTER)
@@ -59,16 +58,15 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
// verify no cluster
assert 0 != exists(CLUSTER).ret
- // create an AM which fails to launch within a second
+ // create an AM which fails to launch
File launchReportFile = createAppReportFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
[
- ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
- ARG_OPTION, InternalKeys.CHAOS_MONKEY_DELAY_SECONDS, "1",
- ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "60",
- ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "100",
+ ARG_OPTION, CHAOS_MONKEY_ENABLED, "true",
+ ARG_OPTION, CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
+ Integer.toString(PROBABILITY_PERCENT_100),
],
launchReportFile)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
new file mode 100644
index 0000000..eb825fc
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AppsThroughAgentDemo extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "agent-demo"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testCreateFlex() throws Throwable {
+ assumeAgentTestsEnabled()
+
+ cleanup(APPLICATION_NAME)
+ SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
+ APP_TEMPLATE,
+ APP_RESOURCE)
+
+ logShell(shell)
+
+ ensureApplicationIsUp(APPLICATION_NAME)
+
+ //flex
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_FLEX,
+ APPLICATION_NAME,
+ ARG_COMPONENT,
+ COMMAND_LOGGER,
+ "2"])
+
+ // sleep till the new instance starts
+ sleep(1000 * 10)
+
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
+
+ String amWebUrl = getInfoAmWebUrl(APPLICATION_NAME)
+ log.info("Dumping data from AM Web URL");
+ log.info(amWebUrl.toURL().text);
+
+ ensureRegistryCallSucceeds(APPLICATION_NAME)
+
+ // get log folders
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ APPLICATION_NAME,
+ ARG_LISTEXP])
+ if(!containsString(shell, "container_log_dirs") ||
+ !containsString(shell, "container_work_dirs")) {
+ logShell(shell)
+ assert fail("Should list default exports container_log_dirs or container_work_dirs")
+ }
+
+ // get log folders
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ APPLICATION_NAME,
+ ARG_GETEXP,
+ "container_log_dirs"])
+ if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
+ || !containsString(shell, "\"level\" : \"component\"", 2)) {
+ logShell(shell)
+ assert fail("Should list 2 entries for log folders")
+ }
+
+ // get log folders
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ APPLICATION_NAME,
+ ARG_GETEXP,
+ "container_work_dirs"])
+ if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
+ || !containsString(shell, "\"level\" : \"component\"", 2)) {
+ logShell(shell)
+ assert fail("Should list 2 entries for work folder")
+ }
+
+ // get cl-site config
+ shell = slider(
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ APPLICATION_NAME,
+ ARG_GETCONF,
+ "cl-site",
+ ARG_FORMAT,
+ "json"])
+
+ for (int i = 0; i < 10; i++) {
+ if (shell.getRet() != EXIT_SUCCESS) {
+ println "Waiting for the cl-site to show up"
+ sleep(1000 * 10)
+ shell = slider(
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ APPLICATION_NAME,
+ ARG_GETCONF,
+ "cl-site",
+ ARG_FORMAT,
+ "json"])
+ }
+ }
+ assert shell.getRet() == EXIT_SUCCESS, "cl-site should be retrieved"
+ if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) {
+ logShell(shell)
+ assert fail("Should have exported cl-site")
+ }
+
+ assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cd37f57/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index fd8330b..488173b 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -27,6 +27,7 @@ import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
+import org.junit.Before
import org.junit.Test
@CompileStatic
@@ -35,8 +36,13 @@ public class AppsThroughAgentIT extends AgentCommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
- private static String APPLICATION_NAME = "happy-path-with-flex"
+ private static String APPLICATION_NAME = "apps-through-agent"
+ @Before
+ public void prepareCluster() {
+ setupCluster(APPLICATION_NAME)
+ }
+
@After
public void destroyCluster() {
cleanup(APPLICATION_NAME)
[10/50] git commit: SLIDER-584 and convert to utf8 before printing
Posted by st...@apache.org.
SLIDER-584 and convert to utf8 before printing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/05cf8982
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/05cf8982
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/05cf8982
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 05cf898288af14da01a0481c2ad5d69c6fcb19dd
Parents: 514d87e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 13:27:05 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 13:27:05 2014 +0000
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/05cf8982/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index e67a6ff..79933bb 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -173,6 +173,7 @@ def print_output(name, src, toStdErr):
# closedown: read remainder of stream
c = src.read(1)
while c!="" :
+ c = c.decode('utf-8')
out(toStdErr, c)
if c == "\n":
flush(toStdErr)
[15/50] git commit: SLIDER-586: negative node count checks on AM
Posted by st...@apache.org.
SLIDER-586: negative node count checks on AM
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fbca62d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fbca62d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fbca62d8
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: fbca62d883c7b1bd8327ca44d06e7e7e5b8f284b
Parents: a19c505
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 15:34:18 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 15:34:18 2014 +0000
----------------------------------------------------------------------
.../TriggerClusterTeardownException.java | 3 +-
.../slideram/SliderAMClientProvider.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 46 ++++++++++++++++----
.../agent/AgentMiniClusterTestBase.groovy | 2 +-
.../slider/providers/agent/TestAgentEcho.groovy | 33 ++++++++++----
.../appstate/TestMockAppStateFlexing.groovy | 24 ++++++++++
6 files changed, 89 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbca62d8/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
index d08b33a..bb9f430 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/TriggerClusterTeardownException.java
@@ -29,8 +29,7 @@ public class TriggerClusterTeardownException extends SliderException {
private final FinalApplicationStatus finalApplicationStatus;
public TriggerClusterTeardownException(int code,
- String message,
- FinalApplicationStatus finalApplicationStatus,
+ FinalApplicationStatus finalApplicationStatus, String message,
Object... args) {
super(code, message, args);
this.finalApplicationStatus = finalApplicationStatus;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbca62d8/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 125746d..5ce0a78 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -150,7 +150,7 @@ public class SliderAMClientProvider extends AbstractClientProvider
int instances = mapOperations.getOptionInt(COMPONENT_INSTANCES, 0);
if (instances < 0) {
throw new BadClusterStateException(
- "Component %s has invalid instance count: %d",
+ "Component %s has negative instance count: %d",
mapOperations.name,
instances);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbca62d8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 31658bc..a69a60d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -678,11 +678,10 @@ public class AppState {
String role = roleStatus.getName();
MapOperations comp =
resources.getComponent(role);
- int desiredInstanceCount =
- resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+ int desiredInstanceCount = getDesiredInstanceCount(resources, role);
if (desiredInstanceCount == 0) {
- log.warn("Role {} has 0 instances specified", role);
- }
+ log.info("Role {} has 0 instances specified", role);
+ }
if (currentDesired != desiredInstanceCount) {
log.info("Role {} flexed from {} to {}", role, currentDesired,
desiredInstanceCount);
@@ -698,13 +697,36 @@ public class AppState {
log.info("Adding new role {}", name);
ProviderRole dynamicRole = createDynamicProviderRole(name,
resources.getComponent(name));
- buildRole(dynamicRole);
+ RoleStatus roleStatus = buildRole(dynamicRole);
+ roleStatus.setDesired(getDesiredInstanceCount(resources, name));
roleHistory.addNewProviderRole(dynamicRole);
}
}
}
/**
+ * Get the desired instance count of a role, rejecting negative values
+ * @param resources resource map
+ * @param role role name
+ * @return the instance count
+ * @throws BadConfigException if the count is negative
+ */
+ private int getDesiredInstanceCount(ConfTreeOperations resources,
+ String role) throws BadConfigException {
+ int desiredInstanceCount =
+ resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+
+ if (desiredInstanceCount < 0) {
+ log.error("Role {} has negative desired instances : {}", role,
+ desiredInstanceCount);
+ throw new BadConfigException(
+ "Negative instance count (%) requested for component %s",
+ desiredInstanceCount, role);
+ }
+ return desiredInstanceCount;
+ }
+
+ /**
* Add knowledge of a role.
* This is a build-time operation that is not synchronized, and
* should be used while setting up the system state -before servicing
@@ -1592,10 +1614,9 @@ public class AppState {
if (failures > threshold) {
throw new TriggerClusterTeardownException(
SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
- ErrorStrings.E_UNSTABLE_CLUSTER +
+ FinalApplicationStatus.FAILED, ErrorStrings.E_UNSTABLE_CLUSTER +
" - failed with component %s failing %d times (%d in startup);" +
" threshold is %d - last failure: %s",
- FinalApplicationStatus.FAILED,
role.getName(),
role.getFailed(),
role.getStartFailed(),
@@ -1651,9 +1672,18 @@ public class AppState {
expected = role.getDesired();
}
- log.info("Reviewing {}", role);
+ log.info("Reviewing {} : expected {}", role, expected);
checkFailureThreshold(role);
+ if (expected < 0 ) {
+ // negative value: fail
+ throw new TriggerClusterTeardownException(
+ SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
+ FinalApplicationStatus.FAILED,
+ "Negative component count of %d desired for component %s",
+ expected, role);
+ }
+
if (delta > 0) {
log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
delta, expected);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbca62d8/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 7786c41..c2ea54a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -140,7 +140,7 @@ extends YarnZKMiniClusterTestBase {
}
/**
- * Create an AM without a master
+ * Create a standalone AM
* @param clustername AM name
* @param size # of nodes
* @param deleteExistingData should any existing cluster data be deleted
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbca62d8/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index f67ee92..f40d5a7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -26,6 +26,7 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
import org.junit.Test
import static org.apache.slider.common.params.Arguments.*
@@ -38,6 +39,28 @@ import static org.apache.slider.providers.agent.AgentKeys.*
@Slf4j
class TestAgentEcho extends AgentTestBase {
+ File slider_core
+ String echo_py
+ File echo_py_path
+ File app_def_path
+ String agt_ver
+ File agt_ver_path
+ String agt_conf
+ File agt_conf_path
+
+ @Before
+ public void setupArtifacts() {
+ slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ echo_py = "echo.py"
+ echo_py_path = new File(slider_core, echo_py)
+ app_def_path = new File(app_def_pkg_path)
+ agt_ver = "version"
+ agt_ver_path = new File(slider_core, agt_ver)
+ agt_conf = "agent.ini"
+ agt_conf_path = new File(slider_core, agt_conf)
+
+ }
+
@Override
void checkTestAssumptions(YarnConfiguration conf) {
@@ -53,14 +76,6 @@ class TestAgentEcho extends AgentTestBase {
true,
false)
- File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- String echo_py = "echo.py"
- File echo_py_path = new File(slider_core, echo_py)
- File app_def_path = new File(app_def_pkg_path)
- String agt_ver = "version"
- File agt_ver_path = new File(slider_core, agt_ver)
- String agt_conf = "agent.ini"
- File agt_conf_path = new File(slider_core, agt_conf)
assert echo_py_path.exists()
assert app_def_path.exists()
assert agt_ver_path.exists()
@@ -102,7 +117,7 @@ class TestAgentEcho extends AgentTestBase {
sliderClient.flex(clustername, [(role): -1]);
fail("expected an exception")
} catch (BadClusterStateException e) {
- assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "")
+ assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fbca62d8/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index a7bf068..1db500b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -112,5 +113,28 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
}
+ @Test
+ public void testFlexNegative() throws Throwable {
+ int r0 = 6
+ int r1 = 0
+ int r2 = 0
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+ log.info("shrinking cluster")
+ role0Status.desired = -2
+ List<AppState.NodeCompletionResult> completionResults = []
+ try {
+ createStartAndStopNodes(completionResults)
+ fail("expected an exception")
+ } catch (TriggerClusterTeardownException e) {
+ }
+
+ }
+
}
[28/50] git commit: SLIDER-570: AgentFailuresIT
Posted by st...@apache.org.
SLIDER-570: AgentFailuresIT
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6e171c77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6e171c77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6e171c77
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 6e171c77f11bf196f56b05d34c6ffb4523476fbd
Parents: e7df654
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 23:06:25 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:50 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/funtest/framework/CommandTestBase.groovy | 4 ++--
.../org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e171c77/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 0a7b295..fc74129 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -1051,14 +1051,14 @@ abstract class CommandTestBase extends SliderTestUtils {
return requested
}
- boolean hasRequestedContainerCountReached(Map<String, String> args) {
+ Outcome hasRequestedContainerCountReached(Map<String, String> args) {
String application = args['application']
String role = args['role']
int expectedCount = args['limit'].toInteger();
int requestedCount = queryRequestedCount(application, role)
log.debug("requested count = $requestedCount; expected=$expectedCount")
- return requestedCount >= expectedCount
+ return Outcome.fromBool(requestedCount >= expectedCount)
}
void expectContainerRequestedCountReached(String application, String role, int limit) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6e171c77/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index d5be0f8..0410881 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -66,7 +66,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2)
sleep(1000 * 20)
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
assertInYarnState(appId, YarnApplicationState.RUNNING)
[08/50] git commit: SLIDER-584 flush output after every line and at
the end of the run
Posted by st...@apache.org.
SLIDER-584 flush output after every line and at the end of the run
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2826027e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2826027e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2826027e
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 2826027e260b9d278769eb22a87d37238eb4419a
Parents: 035dbe7
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 13:18:03 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 13:18:03 2014 +0000
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2826027e/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 38ff68d..e67a6ff 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -122,6 +122,17 @@ def out(toStdErr, text) :
else:
sys.stdout.write(text)
+def flush(toStdErr) :
+ """
+ Flush the output stream
+ :param toStdErr: flag set if stderr is to be the dest
+ :return:
+ """
+ if toStdErr:
+ sys.stderr.flush()
+ else:
+ sys.stdout.flush()
+
def read(pipe, line):
"""
read a char, append to the listing if there is a char that is not \n
@@ -147,6 +158,7 @@ def print_output(name, src, toStdErr):
Relay the output stream to stdout line by line
:param name:
:param src: source stream
+ :param toStdErr: flag set if stderr is to be the dest
:return:
"""
@@ -156,13 +168,16 @@ def print_output(name, src, toStdErr):
(line, done) = read(src, line)
if done:
out(toStdErr, line + "\n")
+ flush(toStdErr)
line = ""
# closedown: read remainder of stream
c = src.read(1)
while c!="" :
out(toStdErr, c)
+ if c == "\n":
+ flush(toStdErr)
c = src.read(1)
-
+ flush(toStdErr)
src.close()
[18/50] git commit: SLIDER-526. Create a python wrapper and replace
the shell script for slider (Thomas Liu via smohanty)
Posted by st...@apache.org.
SLIDER-526. Create a python wrapper and replace the shell script for slider (Thomas Liu via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d03bf28e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d03bf28e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d03bf28e
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: d03bf28e84b155332f4b78ff3ff412c9982d2ef4
Parents: fdcc5c2
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Oct 30 16:30:31 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Oct 30 16:30:31 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 84 ++++---------------------
slider-assembly/src/main/scripts/slider.py | 8 ---
2 files changed, 13 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d03bf28e/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index 51ec060..9646694 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,78 +16,20 @@
# limitations under the License.
-# this is the shell script to start Slider deploying an application
+# this is the wrapper shell script to invoke the Slider.py script in the same folder
# Usage: slider <action> <commands>
-# The env variable SLIDER_JVM_OPTS can be used to override
-# the default JVM opts
+import os
+import sys
+import subprocess
-function usage
-{
- echo "Usage: slider <action> <arguments>"
- echo ""
-}
+ON_POSIX = 'posix' in sys.builtin_module_names
-# Slider works out its own location
-this="${BASH_SOURCE-$0}"
-bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
+currDir = os.path.dirname(os.path.realpath(__file__))
+args = sys.argv[1:]
+cmd = [currDir + "/slider.py"]
+cmd.extend(args)
+finalCmd = " ".join(cmd)
-# lib directory is one up; it is expected to contain
-# slider.jar and any other dependencies that are not in the
-# standard Hadoop classpath
-
-slider_home="${bindir}/.."
-slider_home=`cd -P "${slider_home}" && pwd -P`
-
-libdir="${slider_home}/lib"
-libdir=`cd -P "${libdir}" && pwd -P`
-
-
-confdir="${slider_home}/conf"
-
-# normalize the conf dir so it can be passed down
-confdir=`cd -P "${confdir}" && pwd -P`
-confdir=${SLIDER_CONF_DIR:-$confdir}
-
-
-if [ -f "${confdir}/slider-env.sh" ]; then
- . "${confdir}/slider-env.sh"
-fi
-
-slider_confdir_opts="-Dslider.confdir=${confdir}"
-libdir_jvm_opts="-Dslider.libdir=${libdir}"
-
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m"
-slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
-
-# allow for an extra classpath
-slider_classpath_extra=${SLIDER_CLASSPATH_EXTRA:-""}
-hadoop_conf_dir=${HADOOP_CONF_DIR:-""}
-
-slider_classpath="${libdir}/*:${confdir}:${slider_classpath_extra}:${hadoop_conf_dir}"
-
-launcher=org.apache.slider.Slider
-
-export CLASSPATH="${slider_classpath}"
-
-#echo "slider_home = \"${slider_home}\""
-#echo "slider_jvm_opts = \"${slider_jvm_opts}\""
-#echo "classpath = \"${slider_classpath}\""
-#echo ""
-
-#echo "command is java ${slider_jvm_opts} -classpath \"${slider_classpath}\" ${launcher} $@"
-
-#echo ""
-#echo ""
-
-if [ -n "${JAVA_HOME}" ]; then
- javabin=${JAVA_HOME}/bin/
-else
- javabin=""
-fi
-
-#echo $javabin
-#type ${javabin}java 2>&1
-
-exec ${javabin}java ${slider_jvm_opts} ${slider_confdir_opts} ${libdir_jvm_opts} ${launcher} $@
+result = subprocess.call(finalCmd, shell=True)
+sys.exit(result)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d03bf28e/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 02b0355..26c9adc 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -261,19 +261,11 @@ def java(classname, args, classpath, jvm_opts_list):
return runProcess(commandline)
-def usage():
- print "Usage: slider <action> <arguments>"
- return 1
-
-
def main():
"""
Slider main method
:return: exit code of the process
"""
- if len(sys.argv)==1 :
- return usage()
- # print "stdout encoding: "+ sys.stdout.encoding
args = sys.argv[1:]
slider_home = sliderDir()
libdir = dirMustExist(libDir(slider_home))
[20/50] git commit: SLIDER-574 print log message when no service
keytabs are localized
Posted by st...@apache.org.
SLIDER-574 print log message when no service keytabs are localized
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e23e680f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e23e680f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e23e680f
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: e23e680fe8e329410126ff4a83ad004ce91b528a
Parents: 1a071e3
Author: Jon Maron <jm...@hortonworks.com>
Authored: Thu Oct 30 21:31:41 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Thu Oct 30 21:31:41 2014 -0400
----------------------------------------------------------------------
.../apache/slider/providers/agent/AgentProviderService.java | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e23e680f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 934e5f0..4720ced 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -521,9 +521,11 @@ public class AgentProviderService extends AbstractProviderService implements
getClusterName());
FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(keytabDirPath);
LocalResource keytabRes;
+ boolean serviceKeytabsDeployed = false;
for (FileStatus keytab : keytabs) {
if (!amKeytabName.equals(keytab.getPath().getName())
&& keytab.getPath().getName().endsWith(".keytab")) {
+ serviceKeytabsDeployed = true;
log.info("Localizing keytab {}", keytab.getPath().getName());
keytabRes = fileSystem.createAmResource(keytab.getPath(),
LocalResourceType.FILE);
@@ -532,6 +534,13 @@ public class AgentProviderService extends AbstractProviderService implements
keytabRes);
}
}
+ if (!serviceKeytabsDeployed) {
+ log.warn("No service keytabs for the application have been localized. "
+ + "If the application requires keytabs for secure operation, "
+ + "please ensure that the required keytabs have been uploaded "
+ + "to the folder designated by the property {}: {}",
+ SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR, keytabDirPath);
+ }
}
}
[16/50] git commit: SLIDER-584 don't drop the rest of the line before
the closedown loop kicks in
Posted by st...@apache.org.
SLIDER-584 don't drop the rest of the line before the closedown loop kicks in
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7713527
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7713527
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7713527
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: e7713527ea61c9da7ce2d20cee431abfe2288989
Parents: fbca62d
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 19:11:46 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 19:11:46 2014 +0000
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7713527/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 79933bb..02b0355 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -170,6 +170,7 @@ def print_output(name, src, toStdErr):
out(toStdErr, line + "\n")
flush(toStdErr)
line = ""
+ out(toStdErr, line)
# closedown: read remainder of stream
c = src.read(1)
while c!="" :
[47/50] git commit: SLIDER-598. HBase application package for Windows
need to be tested and validated
Posted by st...@apache.org.
SLIDER-598. HBase application package for Windows need to be tested and validated
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96c9006d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96c9006d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96c9006d
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 96c9006d40c397c5f0244dfbed5b4c5c9eaece7c
Parents: b019271
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Nov 1 17:31:44 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Nov 1 17:31:44 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/appConfig-default.json | 3 +
app-packages/hbase-win/metainfo.xml | 43 ++++++++++++++
.../hbase-win/package/scripts/hbase_rest.py | 62 ++++++++++++++++++++
.../hbase-win/package/scripts/hbase_service.py | 9 +++
.../hbase-win/package/scripts/hbase_thrift.py | 62 ++++++++++++++++++++
.../hbase-win/package/scripts/hbase_thrift2.py | 62 ++++++++++++++++++++
.../hbase-win/package/scripts/params.py | 31 ++++------
app-packages/hbase-win/resources-default.json | 16 +++++
8 files changed, 268 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig-default.json b/app-packages/hbase-win/appConfig-default.json
index 069e2b7..80d20af 100644
--- a/app-packages/hbase-win/appConfig-default.json
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -15,6 +15,9 @@
"site.global.monitor_protocol": "http",
"site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
"site.global.java_library_path": "c:\\hdp\\hadoop\\bin",
+ "site.global.hbase_rest_port": "17000",
+ "site.global.hbase_thrift_port": "9090",
+ "site.global.hbase_thrift2_port": "9091",
"site.hbase-env.hbase_master_heapsize": "1024m",
"site.hbase-env.hbase_regionserver_heapsize": "1024m",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml
index f0c5d4f..da6121d 100644
--- a/app-packages/hbase-win/metainfo.xml
+++ b/app-packages/hbase-win/metainfo.xml
@@ -41,6 +41,18 @@
<name>org.apache.slider.monitor</name>
<value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
</export>
+ <export>
+ <name>org.apache.slider.hbase.rest</name>
+ <value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.hbase.thrift2</name>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.hbase.thrift</name>
+ <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
+ </export>
</exports>
</exportGroup>
</exportGroups>
@@ -80,12 +92,43 @@
<component>
<name>HBASE_REGIONSERVER</name>
<category>SLAVE</category>
+ <minInstanceCount>0</minInstanceCount>
<commandScript>
<script>scripts/hbase_regionserver.py</script>
<scriptType>PYTHON</scriptType>
</commandScript>
</component>
+ <component>
+ <name>HBASE_REST</name>
+ <category>MASTER</category>
+ <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
+ <commandScript>
+ <script>scripts/hbase_rest.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_THRIFT</name>
+ <category>MASTER</category>
+ <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
+ <commandScript>
+ <script>scripts/hbase_thrift.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>HBASE_THRIFT2</name>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports>
+ <commandScript>
+ <script>scripts/hbase_thrift2.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
</components>
<osSpecifics>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_rest.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_rest.py b/app-packages/hbase-win/package/scripts/hbase_rest.py
new file mode 100644
index 0000000..36b51f9
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_rest.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseRest(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='rest')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'rest',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'rest',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-rest.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseRest().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py
index 4a1f103..9a8dc56 100644
--- a/app-packages/hbase-win/package/scripts/hbase_service.py
+++ b/app-packages/hbase-win/package/scripts/hbase_service.py
@@ -33,6 +33,15 @@ def hbase_service(
if name == "regionserver":
heap_size = params.regionserver_heapsize
main_class = "org.apache.hadoop.hbase.regionserver.HRegionServer"
+ if name == "rest":
+ heap_size = params.restserver_heapsize
+ main_class = "org.apache.hadoop.hbase.rest.RESTServer"
+ if name == "thrift":
+ heap_size = params.thriftserver_heapsize
+ main_class = "org.apache.hadoop.hbase.thrift.ThriftServer"
+ if name == "thrift2":
+ heap_size = params.thrift2server_heapsize
+ main_class = "org.apache.hadoop.hbase.thrift2.ThriftServer"
role_user = format("{hbase_user}-{name}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_thrift.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_thrift.py b/app-packages/hbase-win/package/scripts/hbase_thrift.py
new file mode 100644
index 0000000..84bfc62
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_thrift.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/hbase_thrift2.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_thrift2.py b/app-packages/hbase-win/package/scripts/hbase_thrift2.py
new file mode 100644
index 0000000..b72196c
--- /dev/null
+++ b/app-packages/hbase-win/package/scripts/hbase_thrift2.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift2(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift2')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift2',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift2',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift2().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py
index 421b9f5..4f55cd0 100644
--- a/app-packages/hbase-win/package/scripts/params.py
+++ b/app-packages/hbase-win/package/scripts/params.py
@@ -40,40 +40,31 @@ log_dir = config['configurations']['global']['app_log_dir']
input_conf_files_dir = config['configurations']['global']['app_input_conf_dir']
hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
+
+"""
+Read various ports, unused as of now
+"""
rest_port = config['configurations']['global']['hbase_rest_port']
thrift_port = config['configurations']['global']['hbase_thrift_port']
thrift2_port = config['configurations']['global']['hbase_thrift2_port']
+"""
+Compute or read various heap sizes
+"""
master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
+restserver_heapsize = default("configurations/hbase-env/hbase_restserver_heapsize", "512m")
+thriftserver_heapsize = default("configurations/hbase-env/hbase_thriftserver_heapsize", "512m")
+thrift2server_heapsize = default("configurations/hbase-env/hbase_thrift2server_heapsize", "512m")
+
hbase_env_sh_template = config['configurations']['hbase-env']['content']
java_library_path = config['configurations']['global']['java_library_path']
hbase_additional_cp = config['configurations']['global']['hbase_additional_cp']
-master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
-regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file',
- format("{conf_dir}/hbase_regionserver_jaas.conf"))
-master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
-regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
-
-_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
-if security_enabled:
- _hostname_lowercase = config['hostname'].lower()
- master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST', hostname_lowercase)
- regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST', hostname_lowercase)
-
-kinit_path_local = functions.get_kinit_path(
- [default("kinit_path_local", None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-if security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
-else:
- kinit_cmd = ""
-
# log4j.properties
if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])):
log4j_props = config['configurations']['hbase-log4j']['content']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96c9006d/app-packages/hbase-win/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/resources-default.json b/app-packages/hbase-win/resources-default.json
index 4fedf01..91cd640 100644
--- a/app-packages/hbase-win/resources-default.json
+++ b/app-packages/hbase-win/resources-default.json
@@ -13,11 +13,27 @@
"yarn.memory": "256"
},
"slider-appmaster": {
+ "yarn.memory": "256"
},
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
"yarn.memory": "256"
+ },
+ "HBASE_REST": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT": {
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "0",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT2": {
+ "yarn.role.priority": "5",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
}
}
}
[09/50] git commit: Merge branch
'feature/SLIDER-584_CommandEnvironmentIT' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-584_CommandEnvironmentIT' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/514d87e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/514d87e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/514d87e6
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 514d87e6d2592efb7b8d0bbf869e0d0085015def
Parents: 20b071e 2826027
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 13:18:12 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 13:18:12 2014 +0000
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 45 +++++++++++++++++---
.../slider/core/main/ServiceLauncher.java | 2 +
.../slider/funtest/framework/SliderShell.groovy | 35 ++++++++-------
3 files changed, 59 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[45/50] git commit: SLIDER-587: consequences of change: fix now
broken test; improve other tests
Posted by st...@apache.org.
SLIDER-587: consequences of change: fix now broken test; improve other tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b072b9fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b072b9fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b072b9fd
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: b072b9fd6bfc41d29b0d26d3f75ba8dae549b8af
Parents: d03b678
Author: Steve Loughran <st...@apache.org>
Authored: Sat Nov 1 13:11:13 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Nov 1 13:11:13 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/state/RoleHistory.java | 5 ++-
.../TestMockAppStateDynamicHistory.groovy | 23 +++++++++++++
.../TestMockAppStateFlexDynamicRoles.groovy | 35 ++++++++++++--------
.../model/mock/BaseMockAppStateTest.groovy | 7 +++-
.../appmaster/model/mock/MockContainer.groovy | 7 +++-
.../model/mock/MockProviderService.groovy | 6 ++--
.../appmaster/model/mock/MockRoleHistory.groovy | 35 ++++++++++++++++++++
7 files changed, 99 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index f1c0af5..8de4b92 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -131,7 +131,7 @@ public class RoleHistory {
* @throws ArrayIndexOutOfBoundsException
* @throws BadConfigException
*/
- private void checkProviderRole(Map<Integer, RoleStatus> roleStats,
+ protected void checkProviderRole(Map<Integer, RoleStatus> roleStats,
ProviderRole providerRole)
throws BadConfigException {
int index = providerRole.id;
@@ -154,6 +154,8 @@ public class RoleHistory {
*/
public void addNewProviderRole(ProviderRole providerRole)
throws BadConfigException {
+ log.debug("Validating/adding new provider role to role history: {} ",
+ providerRole);
Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
for (ProviderRole role : providerRoles) {
@@ -161,6 +163,7 @@ public class RoleHistory {
}
checkProviderRole(roleStats, providerRole);
+ log.debug("Check successful; adding role");
this.providerRoles.add(providerRole);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
index 7d41012..e06c2cb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -24,9 +24,11 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
@@ -208,4 +210,25 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
def request1 = action1.request
assert request1.nodes
}
+
+ @Test(expected = BadConfigException.class)
+ public void testRoleHistoryRoleAdditions() throws Throwable {
+ MockRoleHistory roleHistory = new MockRoleHistory([])
+ roleHistory.addNewProviderRole(new ProviderRole("one", 1))
+ roleHistory.addNewProviderRole(new ProviderRole("two", 1))
+ roleHistory.dump()
+ fail("should have raised an exception")
+ }
+
+
+ @Test(expected = BadConfigException.class)
+ public void testRoleHistoryRoleStartupConflict() throws Throwable {
+ MockRoleHistory roleHistory = new MockRoleHistory([
+ new ProviderRole("one", 1), new ProviderRole("two", 1)
+ ])
+ roleHistory.dump()
+ fail("should have raised an exception")
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 8308a13..53299dd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -27,10 +27,8 @@ import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -71,7 +69,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
(ResourceKeys.COMPONENT_PRIORITY): "6",
]
- instance.resourceOperations.components["dynamic"] = opts
+ instance.resourceOperations.components["dynamic-6"] = opts
appState.buildInstance(instance,
@@ -98,7 +96,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
(ResourceKeys.COMPONENT_PRIORITY): "7",
]
- cd.components["role4"] = opts
+ cd.components["dynamicAdd7"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
dumpClusterDescription("updated CD", appState.getClusterStatus())
@@ -119,12 +117,17 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
(ResourceKeys.COMPONENT_PRIORITY): "6",
]
- cd.components["role4"] = opts
+ cd.components["conflictingPriority"] = opts
try {
appState.updateResourceDefinitions(cd.confTree);
- dumpClusterDescription("updated CD", appState.getClusterStatus())
- fail("Expected an exception")
+
+ def status = appState.getClusterStatus()
+ dumpClusterDescription("updated CD", status)
+ fail("Expected an exception, got $status")
} catch (BadConfigException expected) {
+ log.info("Expected: {}", expected)
+ log.debug("Expected: {}", expected, expected)
+ // expected
}
}
@@ -149,24 +152,24 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.COMPONENT_PRIORITY): "9",
]
- cd.components["role4"] = opts
+ cd.components["HistorySaveFlexLoad"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
historyWriter.read(fs, history, appState.roleHistory)
}
@Test
- public void testHistoryFlexSaveLoad() throws Throwable {
+ public void testHistoryFlexSaveResetLoad() throws Throwable {
def cd = init()
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.COMPONENT_PRIORITY): "10",
]
- cd.components["role4"] = opts
+ cd.components["HistoryFlexSaveLoad"] = opts
appState.updateResourceDefinitions(cd.confTree);
createAndStartNodes();
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
@@ -185,7 +188,13 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
fs,
historyPath2,
null, null, new SimpleReleaseSelector())
- historyWriter.read(fs, history, appState.roleHistory)
+ // on this read there won't be the right number of roles
+ try {
+ historyWriter.read(fs, history, appState.roleHistory)
+ fail("expected an exception")
+ } catch (IOException e) {
+ assert e.toString().contains("Number of roles")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index c48d7fa..50d7e06 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -33,7 +33,12 @@ import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.RoleStatus
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.apache.slider.test.SliderTestBase
import org.junit.Before
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
index 3eba7c4..9f5d939 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
@@ -18,7 +18,12 @@
package org.apache.slider.server.appmaster.model.mock
-import org.apache.hadoop.yarn.api.records.*
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.api.records.Token
class MockContainer extends Container {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index fc23d54..7efcd96 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -20,13 +20,14 @@ package org.apache.slider.server.appmaster.model.mock
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.service.LifecycleEvent
+import org.apache.hadoop.service.Service.STATE
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.conf.AggregateConf
@@ -34,6 +35,7 @@ import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.launch.ContainerLauncher
+import org.apache.slider.providers.ProviderCompleted
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.actions.QueueAccess
@@ -46,8 +48,6 @@ import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse
import org.apache.slider.server.appmaster.web.rest.agent.Register
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
-import org.apache.slider.providers.ProviderCompleted
-import org.apache.hadoop.service.Service.STATE
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders
class MockProviderService implements ProviderService {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b072b9fd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
new file mode 100644
index 0000000..c521697
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.state.RoleHistory
+
+/**
+ * subclass to enable access to some of the protected methods
+ */
+class MockRoleHistory extends RoleHistory {
+
+ MockRoleHistory(List<ProviderRole> providerRoles) throws BadConfigException {
+ super(providerRoles)
+ }
+
+
+}
[22/50] git commit: SLIDER-579 Add CLI operations/--out args to get
the app reports of launched apps
Posted by st...@apache.org.
SLIDER-579 Add CLI operations/--out args to get the app reports of launched apps
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/25bbfebb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/25bbfebb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/25bbfebb
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 25bbfebb052c59c74950225db759ee93f2f181ba
Parents: 7b63000
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 29 14:59:00 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 54 +++++++++--
.../slider/common/params/ActionCreateArgs.java | 7 ++
.../slider/common/params/ActionExistsArgs.java | 8 +-
.../slider/common/params/ActionLookupArgs.java | 76 +++++++++++++++
.../slider/common/params/ActionThawArgs.java | 8 ++
.../apache/slider/common/params/Arguments.java | 10 +-
.../apache/slider/common/params/ClientArgs.java | 20 +++-
.../common/params/LaunchArgsAccessor.java | 4 +
.../common/params/LaunchArgsDelegate.java | 11 +++
.../common/params/SliderAMCreateAction.java | 7 ++
.../slider/common/params/SliderActions.java | 3 +
.../slider/core/launch/AppMasterLauncher.java | 49 +++++++---
.../slider/core/launch/LaunchedApplication.java | 6 ++
.../launch/SerializedApplicationReport.java | 97 ++++++++++++++++++++
.../core/persist/ApplicationReportSerDeser.java | 57 ++++++++++++
.../slider/core/persist/JsonSerDeser.java | 21 ++++-
.../TestFreezeThawFlexStandaloneAM.groovy | 52 ++++++++++-
17 files changed, 457 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 6ebbcdd..f1f8f97 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.registry.client.api.RegistryOperations;
@@ -55,6 +57,7 @@ import org.apache.hadoop.registry.client.exceptions.NoRecordException;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
@@ -78,6 +81,7 @@ import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionLookupArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
import org.apache.slider.common.params.ActionStatusArgs;
@@ -114,7 +118,9 @@ import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
+import org.apache.slider.core.launch.SerializedApplicationReport;
import org.apache.slider.core.main.RunService;
+import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.registry.SliderRegistryUtils;
@@ -389,6 +395,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionAMSuicideArgs());
} else if (ACTION_LIST.equals(action)) {
exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
+ } else if (ACTION_LOOKUP.equals(action)) {
+ exitCode = actionLookup(serviceArgs.getActionLookupArgs());
} else if (ACTION_REGISTRY.equals(action)) {
exitCode = actionRegistry(serviceArgs.getActionRegistryArgs());
} else if (ACTION_RESOLVE.equals(action)) {
@@ -1006,6 +1014,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.isDebug());
applicationId = launchedApplication.getApplicationId();
+ if (launchArgs.getOutputFile() != null) {
+ // output file has been requested. Get the app report and serialize it
+ ApplicationReport report =
+ launchedApplication.getApplicationReport();
+ SerializedApplicationReport sar = new SerializedApplicationReport(report);
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
+ serDeser.save(sar, launchArgs.getOutputFile());
+ }
int waittime = launchArgs.getWaittime();
if (waittime > 0) {
return waitForAppRunning(launchedApplication, waittime, waittime);
@@ -1374,12 +1390,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setQueue(amQueue);
}
- // Submit the application to the applications manager
- // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
- // Ignore the response as either a valid response object is returned on success
- // or an exception thrown to denote some form of a failure
-
-
// submit the application
LaunchedApplication launchedApplication = amLauncher.submitApplication();
return launchedApplication;
@@ -3300,6 +3310,38 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
print("\n");
}
+ /**
+ * Implement the lookup action.
+ * @param args Action arguments
+ * @return 0 if the entry was found
+ * @throws IOException
+ * @throws YarnException
+ * @throws UnknownApplicationInstanceException if a specific instance
+ * was named but it was not found
+ */
+ @VisibleForTesting
+ public int actionLookup(ActionLookupArgs args)
+ throws IOException, YarnException {
+ verifyBindingsDefined();
+ try {
+ ApplicationId id = ConverterUtils.toApplicationId(args.id);
+ ApplicationReport report = yarnClient.getApplicationReport(id);
+ SerializedApplicationReport sar = new SerializedApplicationReport(report);
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
+ if (args.outputFile != null) {
+ serDeser.save(sar, args.outputFile);
+ } else {
+ println(serDeser.toJson(sar));
+ }
+ } catch (IllegalArgumentException e) {
+ throw new BadCommandArgumentsException(e, "%s : %s", args, e);
+ } catch (ApplicationAttemptNotFoundException notFound) {
+ throw new NotFoundException(notFound, notFound.toString());
+ } catch (ApplicationNotFoundException notFound) {
+ throw new NotFoundException(notFound, notFound.toString());
+ }
+ return EXIT_SUCCESS;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
index cfcfb9d..e70f30a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionCreateArgs.java
@@ -21,6 +21,8 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
@@ -36,6 +38,11 @@ public class ActionCreateArgs extends AbstractClusterBuildingActionArgs
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
@Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
+
+ @Override
public String getRmAddress() {
return launchArgs.getRmAddress();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
index 047d32c..dd1c04b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionExistsArgs.java
@@ -21,6 +21,8 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_EXISTS},
commandDescription = SliderActions.DESCRIBE_ACTION_EXISTS)
@@ -30,6 +32,7 @@ public class ActionExistsArgs extends AbstractActionArgs {
public String getActionName() {
return SliderActions.ACTION_EXISTS;
}
+
@Parameter(names = {ARG_LIVE},
description = "verify that the application is running")
public boolean live;
@@ -37,5 +40,8 @@ public class ActionExistsArgs extends AbstractActionArgs {
@Parameter(names = {ARG_STATE},
description = "verify that the application is in the specific YARN state")
public String state = "";
-
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File out;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
new file mode 100644
index 0000000..3b69e74
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.commons.lang.StringUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.UsageException;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_LOOKUP},
+ commandDescription = SliderActions.DESCRIBE_ACTION_LOOKUP)
+
+public class ActionLookupArgs extends AbstractActionArgs {
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_LOOKUP;
+ }
+
+ public int getMinParams() {
+ return 0;
+ }
+ public int getMaxParams() {
+ return 0;
+ }
+
+ @Parameter(names = {ARG_ID},
+ description = "ID of the container")
+ public String id;
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File outputFile;
+
+ @Override
+ public void validate() throws BadCommandArgumentsException, UsageException {
+ super.validate();
+ if (StringUtils.isEmpty(id)) {
+ throw new BadCommandArgumentsException("Missing mandatory argument "
+ + ARG_ID);
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(SliderActions.ACTION_LOOKUP);
+ if (id!=null) {
+ sb.append(" ");
+ sb.append(ARG_ID).append(" ").append(id);
+ }
+ if (outputFile != null) {
+ sb.append(" ");
+ sb.append(ARG_OUTPUT).append(" ").append(outputFile.getAbsolutePath());
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
index 8408385..b43a14e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
@@ -21,6 +21,8 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_THAW},
commandDescription = SliderActions.DESCRIBE_ACTION_THAW)
public class ActionThawArgs extends AbstractActionArgs implements
@@ -50,4 +52,10 @@ public class ActionThawArgs extends AbstractActionArgs implements
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
+
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 06d9dfb..42efb33 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -49,8 +49,9 @@ public interface Arguments {
*/
String ARG_FILESYSTEM = "--fs";
String ARG_FILESYSTEM_LONG = "--filesystem";
- String ARG_FORMAT = "--format";
+ String ARG_FOLDER = "--folder";
String ARG_FORCE = "--force";
+ String ARG_FORMAT = "--format";
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
@@ -58,6 +59,7 @@ public interface Arguments {
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INTERNAL = "--internal";
+ String ARG_KEYTAB = "--keytab";
String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
@@ -67,19 +69,17 @@ public interface Arguments {
String ARG_MANAGER = "--manager";
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
+ String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
- String ARG_NAME = "--name";
- String ARG_FOLDER = "--folder";
String ARG_OUTPUT = "--out";
String ARG_OUTPUT_SHORT = "-o";
+ String ARG_OVERWRITE = "--overwrite";
String ARG_PACKAGE = "--package";
- String ARG_KEYTAB = "--keytab";
String ARG_PATH = "--path";
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
String ARG_REPLACE_PKG = "--replacepkg";
- String ARG_OVERWRITE = "--overwrite";
String ARG_RESOURCES = "--resources";
String ARG_RES_COMP_OPT = "--rescompopt";
String ARG_RES_COMP_OPT_SHORT = "--rco";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 5a1c049..5ee6cd5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -43,28 +43,31 @@ public class ClientArgs extends CommonArgs {
* This is not bonded to jcommander, it is set up
* after the construction to point to the relevant
* entry
+ *
+ * KEEP IN ALPHABETICAL ORDER
*/
private AbstractClusterBuildingActionArgs buildingActionArgs;
private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
- private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
- private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
- private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
+ private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
private final ActionExistsArgs actionExistsArgs = new ActionExistsArgs();
private final ActionFlexArgs actionFlexArgs = new ActionFlexArgs();
private final ActionFreezeArgs actionFreezeArgs = new ActionFreezeArgs();
+ private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
+ private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
+ private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
private final ActionKillContainerArgs actionKillContainerArgs =
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
+ private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
+ private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
- private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
- private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
public ClientArgs(String[] args) {
@@ -171,6 +174,10 @@ public class ClientArgs extends CommonArgs {
return actionListArgs;
}
+ public ActionLookupArgs getActionLookupArgs() {
+ return actionLookupArgs;
+ }
+
public ActionRegistryArgs getActionRegistryArgs() {
return actionRegistryArgs;
}
@@ -242,6 +249,9 @@ public class ClientArgs extends CommonArgs {
} else if (SliderActions.ACTION_LIST.equals(action)) {
bindCoreAction(actionListArgs);
+ } else if (SliderActions.ACTION_LOOKUP.equals(action)) {
+ bindCoreAction(actionLookupArgs);
+
} else if (SliderActions.ACTION_REGISTRY.equals(action)) {
bindCoreAction(actionRegistryArgs);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
index c36a968..7524053 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java
@@ -18,9 +18,13 @@
package org.apache.slider.common.params;
+import java.io.File;
+
/**
* Launch args for create and start and anything else that can start something
*/
public interface LaunchArgsAccessor extends WaitTimeAccessor {
String getRmAddress();
+
+ File getOutputFile();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
index f0068e2..bc7e94c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java
@@ -20,6 +20,8 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameter;
+import java.io.File;
+
/**
* Any launch-time args
*/
@@ -37,4 +39,13 @@ public class LaunchArgsDelegate extends WaitArgsDelegate implements
public String getRmAddress() {
return rmAddress;
}
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "output file for any application report")
+ public File outputFile;
+
+ @Override
+ public File getOutputFile() {
+ return outputFile;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
index 0d084da..197c22b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java
@@ -22,6 +22,8 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
+import java.io.File;
+
@Parameters(commandNames = {SliderActions.ACTION_CREATE},
commandDescription = SliderActions.DESCRIBE_ACTION_CREATE)
@@ -63,5 +65,10 @@ public class SliderAMCreateAction extends AbstractActionArgs implements
public void setWaittime(int waittime) {
launchArgs.setWaittime(waittime);
}
+
+ @Override
+ public File getOutputFile() {
+ return launchArgs.getOutputFile();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 87e4582..4b92a32 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -36,6 +36,7 @@ public interface SliderActions {
String ACTION_HELP = "help";
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
+ String ACTION_LOOKUP = "lookup";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
@@ -68,6 +69,8 @@ public interface SliderActions {
String DESCRIBE_ACTION_HELP = "Print help information";
String DESCRIBE_ACTION_LIST =
"List running Slider applications";
+ String DESCRIBE_ACTION_LOOKUP =
+ "look up a YARN application";
String DESCRIBE_ACTION_MONITOR =
"Monitor a running application";
String DESCRIBE_ACTION_REGISTRY =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index 303f777..7023c80 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -46,18 +46,19 @@ public class AppMasterLauncher extends AbstractLauncher {
private static final Logger log =
LoggerFactory.getLogger(AppMasterLauncher.class);
- protected final YarnClientApplication application;
- private final String name;
- private final String type;
- private final ApplicationSubmissionContext submissionContext;
- private final ApplicationId appId;
- private final boolean secureCluster;
+ public final YarnClientApplication application;
+ public final String name;
+ public final String type;
+ public final ApplicationSubmissionContext submissionContext;
+ public final ApplicationId appId;
+ public final boolean secureCluster;
private int maxAppAttempts = 0;
private boolean keepContainersOverRestarts = true;
private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
private int priority = 1;
private final Resource resource = Records.newRecord(Resource.class);
private final SliderYarnClientImpl yarnClient;
+ private Long submitTime;
/**
* Build the AM Launcher
@@ -65,10 +66,15 @@ public class AppMasterLauncher extends AbstractLauncher {
* @param type applicatin type
* @param conf hadoop config
* @param fs filesystem binding
- * @param application precreated YARN client app instance
+ * @param yarnClient yarn client
* @param secureCluster is the cluster secure?
- * @param options map of options. All values are extracted in this constructor only
* -the map is not retained.
+ * @param secureCluster flag to indicate secure cluster
+ * @param options map of options. All values are extracted in this constructor only
+ * @param resourceGlobalOptions global options
+ * @param applicationTags any app tags
+ * @throws IOException
+ * @throws YarnException
*/
public AppMasterLauncher(String name,
String type,
@@ -192,7 +198,6 @@ public class AppMasterLauncher extends AbstractLauncher {
completeContainerLaunch();
submissionContext.setAMContainerSpec(containerLaunchContext);
return submissionContext;
-
}
/**
@@ -216,13 +221,35 @@ public class AppMasterLauncher extends AbstractLauncher {
fs.addDelegationTokens(tokenRenewer, credentials);
}
-
+ /**
+ * Submit the application.
+ * @return a launched application representing the submitted application
+ * @throws IOException
+ * @throws YarnException
+ */
public LaunchedApplication submitApplication() throws IOException, YarnException {
completeAppMasterLaunch();
log.info("Submitting application to Resource Manager");
ApplicationId applicationId =
yarnClient.submitApplication(submissionContext);
+ // implicit success; record the time
+ submitTime = System.currentTimeMillis();
return new LaunchedApplication(applicationId, yarnClient);
}
-
+
+ /**
+ * Build a serializable application report. This is a very minimal
+ * report that contains the application Id, name and type —the information
+ * available
+ * @return a data structure which can be persisted
+ */
+ public SerializedApplicationReport createSerializedApplicationReport() {
+ SerializedApplicationReport sar = new SerializedApplicationReport();
+ sar.applicationId = appId.toString();
+ sar.name = name;
+ sar.applicationType = type;
+ sar.queue = queue;
+ sar.submitTime = submitTime;
+ return sar;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
index e5a025c..632e3fd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/LaunchedApplication.java
@@ -95,6 +95,12 @@ public class LaunchedApplication {
return yarnClient.killRunningApplication(applicationId, reason);
}
+ /**
+ * Get the application report of this application
+ * @return an application report
+ * @throws YarnException
+ * @throws IOException
+ */
public ApplicationReport getApplicationReport()
throws YarnException, IOException {
return yarnClient.getApplicationReport(applicationId);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
new file mode 100644
index 0000000..e3d77d0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/SerializedApplicationReport.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.slider.core.persist.ApplicationReportSerDeser;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.io.IOException;
+
+/**
+ * Serialized form of an application report which can be persisted
+ * and then parsed. It can not be converted back into a
+ * real YARN application report
+ *
+ * Useful for testing
+ */
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class SerializedApplicationReport {
+
+ public String applicationId;
+ public String applicationAttemptId;
+ public String name;
+ public String applicationType;
+ public String user;
+ public String queue;
+ public String host;
+ public Integer rpcPort;
+ public String state;
+ public String diagnostics;
+ public String url;
+ /**
+ * This value is non-null only when a report is generated from a submission context.
+ * The YARN {@link ApplicationReport} structure does not propagate this value
+ * from the RM.
+ */
+ public Long submitTime;
+ public Long startTime;
+ public Long finishTime;
+ public String finalStatus;
+ public String origTrackingUrl;
+ public Float progress;
+
+ public SerializedApplicationReport() {
+ }
+
+ public SerializedApplicationReport(ApplicationReport report) {
+ this.applicationId = report.getApplicationId().toString();
+ this.applicationAttemptId = report.getCurrentApplicationAttemptId().toString();
+ this.name = report.getName();
+ this.applicationType = report.getApplicationType();
+ this.user = report.getUser();
+ this.queue = report.getQueue();
+ this.host = report.getHost();
+ this.rpcPort = report.getRpcPort();
+ this.state = report.getYarnApplicationState().toString();
+ this.diagnostics = report.getDiagnostics();
+ this.startTime = report.getStartTime();
+ this.finishTime = report.getFinishTime();
+ this.finalStatus = report.getFinalApplicationStatus().toString();
+ this.progress = report.getProgress();
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return ApplicationReportSerDeser.toString(this);
+ } catch (IOException e) {
+ return super.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
new file mode 100644
index 0000000..a8c72ce
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ApplicationReportSerDeser.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import org.apache.slider.core.launch.SerializedApplicationReport;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.JsonMappingException;
+
+import java.io.IOException;
+
+/**
+ * Persistence of {@link SerializedApplicationReport}
+ *
+ */
+public class ApplicationReportSerDeser
+ extends JsonSerDeser<SerializedApplicationReport> {
+ public ApplicationReportSerDeser() {
+ super(SerializedApplicationReport.class);
+ }
+
+
+ private static final ApplicationReportSerDeser
+ staticinstance = new ApplicationReportSerDeser();
+
+ /**
+ * Convert an instance to a JSON string -sync access to a shared ser/deser
+ * object instance
+ * @param instance object to convert
+ * @return a JSON string description
+ * @throws JsonParseException parse problems
+ * @throws JsonMappingException O/J mapping problems
+ */
+ public static String toString(SerializedApplicationReport instance)
+ throws IOException, JsonGenerationException, JsonMappingException {
+ synchronized (staticinstance) {
+ return staticinstance.toJson(instance);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index f77d6df..2c119ff 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -33,12 +33,13 @@ import org.codehaus.jackson.map.SerializationConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
/**
* Support for marshalling objects to and from JSON.
@@ -126,6 +127,7 @@ public class JsonSerDeser<T> {
* @throws IOException IO problems
* @throws JsonMappingException failure to map from the JSON to this class
*/
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public synchronized T fromResource(String resource)
throws IOException, JsonParseException, JsonMappingException {
InputStream resStream = null;
@@ -206,9 +208,10 @@ public class JsonSerDeser<T> {
/**
- * Save a cluster description to a hadoop filesystem
+ * Save to a hadoop filesystem
* @param fs filesystem
* @param path path
+ * @param instance instance to save
* @param overwrite should any existing file be overwritten
* @throws IOException IO exception
*/
@@ -220,13 +223,23 @@ public class JsonSerDeser<T> {
}
/**
+ * Save an instance to a file
+ * @param instance instance to save
+ * @param file file
+ * @throws IOException
+ */
+ public void save(T instance, File file) throws
+ IOException {
+ writeJsonAsBytes(instance, new FileOutputStream(file));
+ }
+
+ /**
* Write the json as bytes -then close the file
* @param dataOutputStream an outout stream that will always be closed
* @throws IOException on any failure
*/
private void writeJsonAsBytes(T instance,
- DataOutputStream dataOutputStream) throws
- IOException {
+ OutputStream dataOutputStream) throws IOException {
try {
String json = toJson(instance);
byte[] b = json.getBytes(UTF_8);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/25bbfebb/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
index 7e7e52c..70d5ba5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
@@ -24,11 +24,19 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionLookupArgs
+import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.exceptions.NotFoundException
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.main.ServiceLaunchException
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.ApplicationReportSerDeser
import org.junit.Test
/**
@@ -80,12 +88,35 @@ class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase {
localFS.delete(tempConfPath,true)
//now start the cluster
- ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ File appreport = new File("target/$clustername/appreport.json")
+ ServiceLauncher launcher2 = thawCluster(clustername,
+ [Arguments.ARG_OUTPUT, appreport.absolutePath],
+ true);
+
SliderClient newCluster = launcher2.service
addToTeardown(newCluster);
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
+ def sar = serDeser.fromFile(appreport)
+ log.info(sar.toString())
+ assert sar.applicationId != null
+
+ describe("lookup")
+
+ // now via lookup
+ appreport.delete()
+ def lookup1 = new ActionLookupArgs()
+ lookup1.id = sar.applicationId
+
+ assert 0 == newCluster.actionLookup(lookup1)
+ lookup1.outputFile = appreport
+ assert 0 == newCluster.actionLookup(lookup1)
+ sar = serDeser.fromFile(appreport)
+ assert sar.state == YarnApplicationState.RUNNING.toString()
+
newCluster.getClusterDescription(clustername);
+ describe("no change flex")
// while running, flex it with no changes
newCluster.flex(clustername, [:]);
@@ -95,9 +126,28 @@ class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase {
report = newCluster.applicationReport
assert report.finalApplicationStatus == FinalApplicationStatus.KILLED
+ assert 0 == newCluster.actionLookup(lookup1)
+ sar = serDeser.fromFile(appreport)
+ assert sar.finalStatus == FinalApplicationStatus.KILLED.toString()
+
//stop again
assert 0 == clusterActionFreeze(newCluster, clustername)
+ // and add some invalid lookup operations for
+
+ def lookup2 = new ActionLookupArgs()
+ lookup2.id = "invalid"
+ try {
+ newCluster.actionLookup(lookup2)
+ fail("expected $lookup2 to fail")
+ } catch (BadCommandArgumentsException expected) {
+ }
+ try {
+ lookup2.id = "application_1414593568640_0002"
+ newCluster.actionLookup(lookup2)
+ fail("expected $lookup2 to fail")
+ } catch (NotFoundException expected) {
+ }
}
}
[50/50] git commit: upgrade "use a URI in the api" from a SHOULD to a
MUST; fix examples and tests accordingly
Posted by st...@apache.org.
upgrade "use a URI in the api" from a SHOULD to a MUST; fix examples and tests accordingly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b51f5811
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b51f5811
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b51f5811
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: b51f581124644482eeb71e202459b1259cd58f8c
Parents: 0161b59
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 27 18:56:25 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Nov 2 14:14:08 2014 +0000
----------------------------------------------------------------------
.../registry/info/CustomRegistryConstants.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b51f5811/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
index 67b9feb..0a3476c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/CustomRegistryConstants.java
@@ -24,31 +24,31 @@ package org.apache.slider.core.registry.info;
public class CustomRegistryConstants {
public static final String MANAGEMENT_REST_API =
- "org.apache.slider.management";
+ "classpath:org.apache.slider.management";
public static final String REGISTRY_REST_API =
- "org.apache.slider.registry";
+ "classpath:org.apache.slider.registry";
public static final String PUBLISHER_REST_API =
- "org.apache.slider.publisher";
+ "classpath:org.apache.slider.publisher";
public static final String PUBLISHER_CONFIGURATIONS_API =
- "org.apache.slider.publisher.configurations";
+ "classpath:org.apache.slider.publisher.configurations";
public static final String PUBLISHER_EXPORTS_API =
- "org.apache.slider.publisher.exports";
+ "classpath:org.apache.slider.publisher.exports";
public static final String PUBLISHER_DOCUMENTS_API =
- "org.apache.slider.publisher.documents";
+ "classpath:org.apache.slider.publisher.documents";
public static final String AGENT_SECURE_REST_API =
- "org.apache.slider.agents.secure";
+ "classpath:org.apache.slider.agents.secure";
public static final String AGENT_ONEWAY_REST_API =
- "org.apache.slider.agents.oneway";
+ "classpath:org.apache.slider.agents.oneway";
public static final String AM_IPC_PROTOCOL =
- "org.apache.slider.appmaster";
+ "classpath:org.apache.slider.appmaster";
- public static final String WEB_UI = "org.apache.http.UI";
+ public static final String WEB_UI = "http://";
}
[42/50] git commit: Remove extra comma in log statement
Posted by st...@apache.org.
Remove extra comma in log statement
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0a5bb08
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0a5bb08
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0a5bb08
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: b0a5bb088f7c85baba501c423470193c1cf00cdf
Parents: c552af8
Author: tedyu <yu...@gmail.com>
Authored: Fri Oct 31 20:55:24 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Oct 31 20:55:24 2014 -0700
----------------------------------------------------------------------
.../libraries/functions/check_process_status.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0a5bb08/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
index ff359de..b491f66 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
@@ -44,14 +44,14 @@ def check_process_status(pid_file):
if not pid_file:
Logger.warn("pid_file is not valid")
else:
- Logger.info("pid file does not exist {0}".format(pid_file,))
+ Logger.info("pid file does not exist {0}".format(pid_file))
raise ComponentIsNotRunning()
with open(pid_file, "r") as f:
try:
pid = int(f.read())
except:
- Logger.debug("Pid file {0} does not exist".format(pid_file,))
+ Logger.debug("Pid file {0} does not exist".format(pid_file))
raise ComponentIsNotRunning()
if IS_WINDOWS:
[43/50] git commit: SLIDER-598. HBase application package for Windows
Posted by st...@apache.org.
SLIDER-598. HBase application package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a07eaf0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a07eaf0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a07eaf0a
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: a07eaf0a08a6f18ea7672403e369def3080fab89
Parents: b0a5bb0
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Oct 31 22:02:36 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Oct 31 22:02:36 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig-default.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a07eaf0a/app-packages/hbase/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
index a36a693..52587e5 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -8,7 +8,7 @@
"java_home": "/usr/jdk64/jdk1.7.0_67",
"system_configs": "core-site",
- "site.global.app_user": "${USER_NAME}",
+ "site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
"site.global.ganglia_server_host": "${NN_HOST}",
[12/50] git commit: SLIDER-581 moving where the exit codes are set
up/using non superclass exec
Posted by st...@apache.org.
SLIDER-581 moving where the exit codes are set up/using non superclass exec
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/518d7508
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/518d7508
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/518d7508
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 518d7508c239692db60722fb2cbe9805551b66f0
Parents: 2d953ff
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 14:34:33 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 14:34:33 2014 +0000
----------------------------------------------------------------------
.../apache/slider/funtest/framework/SliderShell.groovy | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/518d7508/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index c101cfa..43ac477 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -79,9 +79,14 @@ class SliderShell extends Shell {
List<String> commandLine = buildEnvCommands()
commandLine << command
+ if (org.apache.hadoop.util.Shell.WINDOWS) {
+ // Ensure the errorlevel returned by last call is set for the invoking shell
+ commandLine << "@echo ERRORLEVEL=%ERRORLEVEL%"
+ commandLine << "@exit %ERRORLEVEL%"
+ }
String script = commandLine.join("\n")
log.debug(script)
- super.exec(script);
+ exec(script);
signCorrectReturnCode()
return ret;
}
@@ -251,12 +256,6 @@ class SliderShell extends Shell {
def writer = new PrintWriter(new BufferedOutputStream(proc.out))
writer.println(script)
writer.flush()
- if (org.apache.hadoop.util.Shell.WINDOWS) {
- // Ensure the errorlevel returned by last call is set for the invoking shell
- writer.println("@echo ERRORLEVEL=%ERRORLEVEL%")
- writer.println("@exit %ERRORLEVEL%")
- writer.flush()
- }
writer.close()
}
[06/50] git commit: SLIDER-583 Slider-HA:ClusterConnectivityIT failed
due to connection refused
Posted by st...@apache.org.
SLIDER-583 Slider-HA:ClusterConnectivityIT failed due to connection refused
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/20b071e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/20b071e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/20b071e8
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 20b071e87eda37625d17b53e7eafc40ef9c30363
Parents: 8c2e09f
Author: Gour Saha <go...@apache.org>
Authored: Thu Oct 30 02:40:06 2014 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Thu Oct 30 02:40:06 2014 -0700
----------------------------------------------------------------------
.../slider/funtest/basic/ClusterConnectivityIT.groovy | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/20b071e8/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
index b09e8b7..9826e97 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
@@ -65,13 +65,17 @@ class ClusterConnectivityIT extends CommandTestBase {
@Test
public void testRMTelnet() throws Throwable {
- def rmAddr = SLIDER_CONFIG.getSocketAddr(YarnConfiguration.RM_ADDRESS, "", 0)
- telnet(rmAddr.hostName, rmAddr.port)
+ def isHaEnabled = SLIDER_CONFIG.getTrimmed(YarnConfiguration.RM_HA_ENABLED)
+ // Telnet test is not straight forward for HA setup and is not required
+ // as long as RM binding is tested
+ if (!isHaEnabled) {
+ def rmAddr = SLIDER_CONFIG.getSocketAddr(YarnConfiguration.RM_ADDRESS, "", 0)
+ telnet(rmAddr.hostName, rmAddr.port)
+ }
}
@Test
public void testRMBinding() throws Throwable {
- testRMTelnet()
SliderYarnClientImpl yarnClient = new SliderYarnClientImpl()
try {
SLIDER_CONFIG.setInt("ipc.client.connect.retry.interval",100)
[34/50] git commit: SLIDER-587: building test for dynamic role
placement history
Posted by st...@apache.org.
SLIDER-587: building test for dynamic role placement history
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/22b4b5e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/22b4b5e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/22b4b5e7
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 22b4b5e72aa458f4e7b5cf4741ee3304057d8ed2
Parents: 2ac8428
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 16:54:37 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 16:54:37 2014 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 11 ++--
.../server/appmaster/state/RoleHistory.java | 6 +--
.../server/appmaster/state/RoleInstance.java | 5 ++
.../TestMockAppStateContainerFailure.groovy | 7 ++-
.../TestMockAppStateDynamicRoles.groovy | 54 ++++++++++++++++++--
.../TestMockAppStateFlexDynamicRoles.groovy | 5 +-
.../TestMockAppStateRMOperations.groovy | 3 +-
.../TestMockAppStateRebuildOnAMRestart.groovy | 10 ++--
.../TestMockAppStateRolePlacement.groovy | 4 +-
.../model/mock/BaseMockAppStateTest.groovy | 34 ++++++++++--
.../appmaster/model/mock/MockAppState.groovy | 14 +++++
11 files changed, 130 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index a69a60d..406086a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -574,20 +574,21 @@ public class AppState {
* @return a new provider role
* @throws BadConfigException bad configuration
*/
+ @VisibleForTesting
public ProviderRole createDynamicProviderRole(String name,
MapOperations component) throws
BadConfigException {
String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
- int pri = SliderUtils.parseAndValidate("value of " + name + " " +
+ int priority = SliderUtils.parseAndValidate("value of " + name + " " +
ResourceKeys.COMPONENT_PRIORITY,
priOpt, 0, 1, -1);
- log.info("Role {} assigned priority {}", name, pri);
+ log.info("Role {} assigned priority {}", name, priority);
String placementOpt = component.getOption(
ResourceKeys.COMPONENT_PLACEMENT_POLICY, "0");
int placement = SliderUtils.parseAndValidate("value of " + name + " " +
ResourceKeys.COMPONENT_PLACEMENT_POLICY,
placementOpt, 0, 0, -1);
- return new ProviderRole(name, pri, placement);
+ return new ProviderRole(name, priority, placement);
}
@@ -694,11 +695,13 @@ public class AppState {
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
+ MapOperations component = resources.getComponent(name);
log.info("Adding new role {}", name);
ProviderRole dynamicRole = createDynamicProviderRole(name,
- resources.getComponent(name));
+ component);
RoleStatus roleStatus = buildRole(dynamicRole);
roleStatus.setDesired(getDesiredInstanceCount(resources, name));
+ log.info("New role {}", roleStatus);
roleHistory.addNewProviderRole(dynamicRole);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 2b35ad5..2b0ee18 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -430,9 +430,9 @@ public class RoleHistory {
/**
* Get the nodes for an ID -may be null
* @param id role ID
- * @return list
+ * @return potenially null list
*/
- private LinkedList<NodeInstance> getNodesForRoleId(int id) {
+ private List<NodeInstance> getNodesForRoleId(int id) {
return availableNodes.get(id);
}
@@ -481,7 +481,7 @@ public class RoleHistory {
List<NodeInstance> targets = getNodesForRoleId(roleKey);
int cnt = targets == null ? 0 : targets.size();
- log.debug("There're {} nodes to consider for {}", cnt, role.getName());
+ log.debug("There are {} node(s) to consider for {}", cnt, role.getName());
while (targets != null && !targets.isEmpty() && nodeInstance == null) {
NodeInstance head = targets.remove(0);
if (head.getActiveRoleInstances(roleKey) == 0) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index c6d8f4c..c8ddc6f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -51,8 +51,13 @@ public final class RoleInstance implements Cloneable {
* already been targeted for termination
*/
public boolean released;
+
+ /**
+ * Name of the role
+ */
public String role;
public int roleId;
+
/**
* state from {@link ClusterDescription}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index 9902155..6368a3d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -29,7 +29,12 @@ import org.apache.slider.server.appmaster.actions.ResetFailureWindow
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 136e1ea..5ef639b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -22,12 +22,13 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
@@ -58,7 +59,7 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
@Override
void initApp() {
super.initApp()
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
def instance = factory.newInstanceDefinition(0,0,0)
@@ -90,5 +91,50 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
appState.getRoleHistory().dump();
}
-
+
+
+ @Test
+ public void testDynamicRoleHistory() throws Throwable {
+
+ // snapshot and patch existing spec
+ def resources = ConfTreeOperations.fromInstance(
+ appState.resourcesSnapshot.confTree)
+
+ def name = "dynamic"
+ def dynamicComp = resources.getOrAddComponent(name)
+ int priority = 8
+ int placement = 3
+ dynamicComp.put(ResourceKeys.COMPONENT_PRIORITY, "8")
+ dynamicComp.put(ResourceKeys.COMPONENT_INSTANCES, "1")
+ dynamicComp.put(ResourceKeys.COMPONENT_PLACEMENT_POLICY, "3")
+
+ def component = resources.getComponent(name)
+ String priOpt = component.getMandatoryOption(
+ ResourceKeys.COMPONENT_PRIORITY);
+ int parsedPriority = SliderUtils.parseAndValidate(
+ "value of " + name + " " + ResourceKeys.COMPONENT_PRIORITY,
+ priOpt, 0, 1, -1);
+ assert priority == parsedPriority
+
+ def newRole = appState.createDynamicProviderRole(name, component)
+ assert newRole.id == priority
+
+ appState.updateResourceDefinitions(resources.confTree);
+ assert appState.roleMap[name] != null
+ def mappedRole = appState.roleMap[name]
+ assert mappedRole.id == priority
+
+ def priorityMap = appState.rolePriorityMap
+ assert priorityMap.size() == 4
+ def dynamicProviderRole
+ assert (dynamicProviderRole = priorityMap[priority]) != null
+ assert dynamicProviderRole.id == priority
+
+ // allocate the nodes
+ def allocations = createAndStartNodes()
+ assert allocations.size() == 1
+ RoleInstance ri = allocations[0]
+ assert ri.role == name
+ assert ri.roleId == priority
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 5c9dce9..8308a13 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -26,6 +26,7 @@ import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
@@ -60,7 +61,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
@Override
void initApp() {
super.initApp()
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
def instance = factory.newInstanceDefinition(0, 0, 0)
@@ -174,7 +175,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
//now reset the app state
def historyWorkDir2 = new File("target/history" + testName + "-0002")
def historyPath2 = new Path(historyWorkDir2.toURI())
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index e5ad4ae..ee5eead 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -31,7 +31,8 @@ import org.apache.slider.server.appmaster.operations.CancelRequestOperation
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.operations.RMOperationHandler
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c2783f3..b48a683 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -24,10 +24,14 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.Container
import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
/**
@@ -67,7 +71,7 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
NodeMap nodemap = appState.roleHistory.cloneNodemap()
// now destroy the app state
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
//and rebuild
appState.buildInstance(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index 17ebc31..6df8910 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -27,7 +27,9 @@ import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.*
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleHistoryUtils
+import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 6c0f571..fa54256 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -43,7 +43,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
public static final int RM_MAX_RAM = 4096
public static final int RM_MAX_CORES = 64
MockFactory factory = new MockFactory()
- AppState appState
+ MockAppState appState
MockYarnEngine engine
protected HadoopFS fs
protected SliderFileSystem sliderFileSystem
@@ -86,7 +86,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
- appState = new AppState(new MockRecordFactory())
+ appState = new MockAppState()
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
buildInstanceDefinition(),
@@ -172,7 +172,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
*/
public ContainerStatus containerStatus(ContainerId cid) {
ContainerStatus status = containerStatus(cid,
- LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN)
return status
}
@@ -203,6 +203,20 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
List<AppState.NodeCompletionResult> completionResults) {
List<ContainerId> released = []
List<RoleInstance> instances = createAndSubmitNodes(released)
+ processSubmissionOperations(instances, completionResults, released)
+ return instances
+ }
+
+ /**
+ * Process the start/stop operations from
+ * @param instances
+ * @param completionResults
+ * @param released
+ */
+ public void processSubmissionOperations(
+ List<RoleInstance> instances,
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> released) {
for (RoleInstance instance : instances) {
assert appState.onNodeManagerContainerStarted(instance.containerId)
}
@@ -212,7 +226,6 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
"released",
0
)
- return instances
}
/**
@@ -256,6 +269,19 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
public List<RoleInstance> createAndSubmitNodes(
List<ContainerId> released) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ return submitOperations(ops, released)
+ }
+
+ /**
+ * Process the RM operations and send <code>onContainersAllocated</code>
+ * events to the app state
+ * @param ops
+ * @param released
+ * @return
+ */
+ public List<RoleInstance> submitOperations(
+ List<AbstractRMOperation> ops,
+ List<ContainerId> released) {
List<Container> allocatedContainers = engine.execute(ops, released)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> operations = []
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22b4b5e7/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 25c957e..ad85b89 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -17,13 +17,27 @@
package org.apache.slider.server.appmaster.model.mock
+import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.state.AbstractRecordFactory
import org.apache.slider.server.appmaster.state.AppState
+/**
+ * Extended app state that makes more things public
+ */
class MockAppState extends AppState {
public MockAppState(AbstractRecordFactory recordFactory) {
super(recordFactory);
}
+ /**
+ * Instance with a mock record factory
+ */
+ public MockAppState() {
+ super(new MockRecordFactory());
+ }
+
+ public Map<String, ProviderRole> getRoleMap() {
+ return super.roleMap;
+ }
}
[40/50] git commit: SLIDER-598. HBase application package for Windows
Posted by st...@apache.org.
SLIDER-598. HBase application package for Windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/48d6bd2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/48d6bd2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/48d6bd2d
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 48d6bd2d62716b77210506bd07bd9754b26658ce
Parents: 9c2080b
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Oct 31 19:04:40 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Oct 31 19:04:40 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/README.txt | 2 +-
app-packages/hbase-win/appConfig-default.json | 4 ++--
app-packages/hbase-win/package/scripts/hbase.py | 5 -----
app-packages/hbase-win/package/scripts/hbase_service.py | 1 -
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/hbase/appConfig-secured-default.json | 2 +-
slider-agent/src/main/python/agent/ActionQueue.py | 3 ++-
.../libraries/functions/check_process_status.py | 7 ++++++-
8 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/app-packages/hbase-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
index 4e0e30a..a413e76 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -27,7 +27,7 @@ with appropriate parameters.
Command:
mvn clean package -Phbase-app-package-win -Dpkg.version=<version>
- -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
+ -Dpkg.name=<file name of app zip file> -Dpkg.src=<folder location where the pkg is available>
Example:
mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/app-packages/hbase-win/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig-default.json b/app-packages/hbase-win/appConfig-default.json
index a802fed..069e2b7 100644
--- a/app-packages/hbase-win/appConfig-default.json
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -14,12 +14,12 @@
"site.global.user_group": "hadoop",
"site.global.monitor_protocol": "http",
"site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
- "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
+ "site.global.java_library_path": "c:\\hdp\\hadoop\\bin",
"site.hbase-env.hbase_master_heapsize": "1024m",
"site.hbase-env.hbase_regionserver_heapsize": "1024m",
"site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
- "site.hbase-site.hbase.superuser": "yarn",
+ "site.hbase-site.hbase.superuser": "hadoop",
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/app-packages/hbase-win/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase.py b/app-packages/hbase-win/package/scripts/hbase.py
index 5ae64eb..0962149 100644
--- a/app-packages/hbase-win/package/scripts/hbase.py
+++ b/app-packages/hbase-win/package/scripts/hbase.py
@@ -28,20 +28,15 @@ def hbase(name=None # 'master' or 'regionserver'
import params
Directory( params.conf_dir,
- owner = params.hbase_user,
- group = params.user_group,
recursive = True,
content = params.input_conf_files_dir
)
Directory (params.tmp_dir,
- owner = params.hbase_user,
recursive = True
)
Directory (os.path.join(params.local_dir, "jars"),
- owner = params.hbase_user,
- group = params.user_group,
recursive = True
)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/app-packages/hbase-win/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py
index c02526c..4a1f103 100644
--- a/app-packages/hbase-win/package/scripts/hbase_service.py
+++ b/app-packages/hbase-win/package/scripts/hbase_service.py
@@ -41,7 +41,6 @@ def hbase_service(
process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}")
Execute(process_cmd,
- user=params.hbase_user,
logoutput=False,
wait_for_finish=False,
pid_file=pid_file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/app-packages/hbase/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
index 44e7c82..a36a693 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -40,7 +40,7 @@
},
"components": {
"slider-appmaster": {
- "jvm.heapsize": "256M"
+ "jvm.heapsize": "1024M"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/app-packages/hbase/appConfig-secured-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-secured-default.json b/app-packages/hbase/appConfig-secured-default.json
index ebe4137..19bc04a 100644
--- a/app-packages/hbase/appConfig-secured-default.json
+++ b/app-packages/hbase/appConfig-secured-default.json
@@ -54,7 +54,7 @@
},
"components": {
"slider-appmaster": {
- "jvm.heapsize": "256M",
+ "jvm.heapsize": "1024M",
"slider.hdfs.keytab.dir": ".slider/keytabs/hbase",
"slider.am.login.keytab.name": "${USER_NAME}.headless.keytab",
"slider.keytab.principal.name": "${USER_NAME}"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index b19f65f..4cb5de7 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -151,9 +151,10 @@ class ActionQueue(threading.Thread):
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
store_command = False
if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
- logger.info("Component has indicated auto-restart. Saving details from START command.")
store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
+ if store_command:
+ logger.info("Component has indicated auto-restart. Saving details from START command.")
# running command
commandresult = self.customServiceOrchestrator.runCommand(command,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48d6bd2d/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
index e17b5f7..ff359de 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/check_process_status.py
@@ -41,12 +41,17 @@ def check_process_status(pid_file):
@param pid_file: path to service pid file
"""
if not pid_file or not os.path.isfile(pid_file):
+ if not pid_file:
+ Logger.warn("pid_file is not valid")
+ else:
+ Logger.info("pid file does not exist {0}".format(pid_file,))
raise ComponentIsNotRunning()
+
with open(pid_file, "r") as f:
try:
pid = int(f.read())
except:
- Logger.debug("Pid file {0} does not exist".format(pid_file))
+ Logger.debug("Pid file {0} does not exist".format(pid_file,))
raise ComponentIsNotRunning()
if IS_WINDOWS:
[27/50] git commit: SLIDER-570 handling of launch failures
Posted by st...@apache.org.
SLIDER-570 handling of launch failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/73462659
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/73462659
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/73462659
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 734626596c0af041c53637ac8053eb50d0a8d169
Parents: 517042f
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 29 20:49:39 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../funtest/framework/CommandTestBase.groovy | 95 ++++++++++++++++++--
.../slider/funtest/framework/SliderShell.groovy | 4 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 6 +-
.../lifecycle/AgentLaunchFailureIT.groovy | 95 ++++++++++++++++++++
.../funtest/lifecycle/AgentRegistryIT.groovy | 6 +-
5 files changed, 189 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73462659/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 7b50c60..7928642 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -30,12 +30,15 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.SerializedApplicationReport
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.client.SliderClient
+import org.apache.slider.core.persist.ApplicationReportSerDeser
import org.apache.slider.test.SliderTestUtils
import org.junit.Before
import org.junit.BeforeClass
@@ -369,6 +372,20 @@ abstract class CommandTestBase extends SliderTestUtils {
slider(cmd)
}
+ static SliderShell lookup(int result, String id, File out) {
+ assert id
+ def commands = [ACTION_LOOKUP, ARG_ID, id]
+ if (out) commands += [ARG_OUTPUT, out.absolutePath]
+ slider(result, commands)
+ }
+
+ static SliderShell lookup(String id, File out) {
+ assert id
+ def commands = [ACTION_LOOKUP, ARG_ID, id]
+ if (out) commands += [ARG_OUTPUT, out.absolutePath]
+ slider(commands)
+ }
+
static SliderShell list(int result, Collection<String> commands =[]) {
slider(result, [ACTION_LIST] + commands )
}
@@ -608,11 +625,18 @@ abstract class CommandTestBase extends SliderTestUtils {
String name,
String appTemplate,
String resourceTemplate,
- List<String> extraArgs=[]) {
+ List<String> extraArgs = [],
+ File launchReport = null) {
+
+ if (!launchReport) {
+ launchReport = createAppReportFile()
+ }
+
List<String> commands = [
ACTION_CREATE, name,
ARG_TEMPLATE, appTemplate,
ARG_RESOURCES, resourceTemplate,
+ ARG_OUTPUT, launchReport.absolutePath,
ARG_WAIT, Integer.toString(THAW_WAIT_TIME)
]
@@ -633,20 +657,35 @@ abstract class CommandTestBase extends SliderTestUtils {
shell.execute()
if (!shell.execute()) {
// app has failed.
-
+
// grab the app report of the last known instance of this app
// which may not be there if it was a config failure; may be out of date
// from a previous run
- log.error("Launch failed with exit code ${shell.ret}.\nLast instance of $name:")
- slider([ACTION_LIST, name, ARG_VERBOSE]).dumpOutput()
-
- // trigger the assertion failure
- shell.assertExitCode(EXIT_SUCCESS)
+ log.error(
+ "Launch failed with exit code ${shell.ret}")
+ shell.dumpOutput()
+
+ // now grab that app report if it is there
+ def appReport = maybeLookupFromLaunchReport(launchReport)
+ String extraText = ""
+ if (appReport) {
+ log.error("Application report:\n$appReport")
+ extraText = appReport.diagnostics
+ }
+
+ fail("Application Launch Failure, exit code ${shell.ret}\n${extraText}")
}
-
return shell
}
+ public File createAppReportFile() {
+ File reportFile = File.createTempFile(
+ "launch",
+ ".json",
+ new File("target"))
+ return reportFile
+ }
+
/**
* If the option is not null/empty, add the command and the option
* @param args arg list being built up
@@ -662,7 +701,47 @@ abstract class CommandTestBase extends SliderTestUtils {
}
return args
}
+
+ public SerializedApplicationReport maybeLoadAppReport(File reportFile) {
+ if (reportFile.exists() && reportFile.length()> 0) {
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
+ def report = serDeser.fromFile(reportFile)
+ return report
+ }
+ return null;
+ }
+
+ public SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
+ def report = maybeLoadAppReport(launchReport)
+ if (report) {
+ return lookupApplication(report.applicationId)
+ } else {
+ return null
+ }
+ }
+
+ /**
+ * Lookup an application, return null if loading failed
+ * @param id application ID
+ * @return an application report or null
+ */
+ public SerializedApplicationReport lookupApplication(String id) {
+ File reportFile = createAppReportFile();
+ try {
+ def shell = lookup(id, reportFile)
+ if (shell.ret) {
+ return maybeLoadAppReport(reportFile)
+ } else {
+ log.warn("Lookup operation failed:\n" + shell.dumpOutput())
+ return null
+ }
+ } finally {
+ reportFile.delete()
+
+ }
+ }
+
public Path buildClusterPath(String clustername) {
return new Path(
clusterFS.homeDirectory,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73462659/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 43ac477..31830d9 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -223,11 +223,11 @@ class SliderShell extends Shell {
* if not the output is printed and an assertion is raised
* @param errorCode expected error code
*/
- public void assertExitCode(int errorCode) {
+ public void assertExitCode(int errorCode, String extra="") {
if (this.ret != errorCode) {
dumpOutput()
throw new SliderException(ret,
- "Expected exit code of command ${command} : ${errorCode} - actual=${ret}")
+ "Expected exit code of command ${command} : ${errorCode} - actual=${ret} $extra")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73462659/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index a4eb1a2..3847e3f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -50,9 +50,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
cleanup(APPLICATION_NAME)
- def shell = createTemplatedSliderApplication( APPLICATION_NAME,
- APP_TEMPLATE2,
- APP_RESOURCE)
+ def shell = createTemplatedSliderApplication(APPLICATION_NAME,
+ APP_TEMPLATE2,
+ APP_RESOURCE)
logShell(shell)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73462659/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
new file mode 100644
index 0000000..ce1e0f1
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.Endpoint
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.slider.api.InternalKeys
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+
+@CompileStatic
+@Slf4j
+public class AgentLaunchFailureIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+ static String CLUSTER = "test-agent-launchfail"
+
+ static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(CLUSTER)
+ }
+
+ @Test
+ public void testAgentLaunchFailure() throws Throwable {
+ describe("Create a failing cluster and validate failure logic")
+
+ // create an AM which fails to launch within a second
+ File launchReportFile = createAppReportFile();
+ SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+ APP_TEMPLATE,
+ APP_RESOURCE2,
+ [
+ ARG_INTERNAL, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
+ ARG_INTERNAL, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "1",
+ ARG_INTERNAL, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "100",
+ ],
+ launchReportFile)
+
+ maybeLookupFromLaunchReport(launchReportFile)
+ ensureApplicationIsUp(CLUSTER)
+
+
+ //stop
+ freeze(0, CLUSTER,
+ [
+ ARG_FORCE,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
+
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73462659/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index 50da8ae..16e65fa 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -49,9 +49,7 @@ public class AgentRegistryIT extends AgentCommandTestBase
@Before
public void prepareCluster() {
setupCluster(CLUSTER)
-
-
- }
+ }
@After
public void destroyCluster() {
@@ -59,7 +57,7 @@ public class AgentRegistryIT extends AgentCommandTestBase
}
@Test
- public void testAgentClusterLifecycle() throws Throwable {
+ public void testAgentRegistry() throws Throwable {
describe("Create a 0-role cluster and make registry queries against it")
// sanity check to verify the config is correct
[11/50] git commit: SLIDER-581 ensure that exit codes come back on
funtests on windows
Posted by st...@apache.org.
SLIDER-581 ensure that exit codes come back on funtests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2d953fff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2d953fff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2d953fff
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 2d953fff1f3d65dc19e4d0fb41ea9fd394de00d5
Parents: 05cf898
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 14:23:34 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 14:23:34 2014 +0000
----------------------------------------------------------------------
.../slider/funtest/framework/SliderShell.groovy | 15 +++++--
.../funtest/commands/CommandExitCodesIT.groovy | 43 ++++++++++++++++++++
.../funtest/commands/ListCommandIT.groovy | 2 +-
.../commands/UnknownClusterOperationsIT.groovy | 2 -
4 files changed, 55 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d953fff/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 15bf339..c101cfa 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -240,18 +240,25 @@ class SliderShell extends Shell {
* @return Shell object for chaining
*/
Shell exec(Object... args) {
- def proc = "$shell".execute()
+ Process proc = "$shell".execute()
script = args.join("\n")
LOG.debug("${shell} << __EOT__\n${script}\n__EOT__");
+ ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
+ ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
+ proc.consumeProcessOutput(baosOut, baosErr)
Thread.start {
def writer = new PrintWriter(new BufferedOutputStream(proc.out))
writer.println(script)
+ writer.flush()
+ if (org.apache.hadoop.util.Shell.WINDOWS) {
+ // Ensure the errorlevel returned by last call is set for the invoking shell
+ writer.println("@echo ERRORLEVEL=%ERRORLEVEL%")
+ writer.println("@exit %ERRORLEVEL%")
+ writer.flush()
+ }
writer.close()
}
- ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
- ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
- proc.consumeProcessOutput(baosOut, baosErr)
proc.waitFor()
ret = proc.exitValue()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d953fff/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
new file mode 100644
index 0000000..c49c738
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class CommandExitCodesIT extends CommandTestBase {
+
+ @Test
+ public void testHelp() throws Throwable {
+ slider(0, ["help"])
+ }
+
+ @Test
+ public void testStopWithoutCluster() throws Throwable {
+ slider(LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, ["stop"])
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d953fff/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
index 7ba7fc0..c65280a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
@@ -36,7 +36,7 @@ public class ListCommandIT extends CommandTestBase {
@Test
public void testListAllLive() throws Throwable {
def shell = list("--live")
- assert shell.ret == 0 || shell.ret ==1
+ assert shell.ret == 0 || shell.ret == 1
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d953fff/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
index 6eaf54a..5515894 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/UnknownClusterOperationsIT.groovy
@@ -21,7 +21,6 @@ package org.apache.slider.funtest.commands
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.funtest.categories.FunctionalTests
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.CommandTestBase
@@ -33,7 +32,6 @@ import org.junit.Test
*/
@CompileStatic
@Slf4j
-@org.junit.experimental.categories.Category(FunctionalTests)
public class UnknownClusterOperationsIT extends CommandTestBase {
public static final String UNKNOWN = "unknown_cluster"
[33/50] git commit: Merge branch
'feature/SLIDER-570-failure-text-in-funtests' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-570-failure-text-in-funtests' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2ac84287
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2ac84287
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2ac84287
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 2ac84287c44b9190ef7e5979a406a125df424f0c
Parents: e23e680 1f98074
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 14:24:54 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 14:24:54 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/InternalKeys.java | 27 +-
.../org/apache/slider/client/SliderClient.java | 166 ++++++---
.../slider/client/SliderYarnClientImpl.java | 18 +-
.../org/apache/slider/common/Constants.java | 1 -
.../slider/common/params/ActionCreateArgs.java | 7 +
.../slider/common/params/ActionDestroyArgs.java | 1 +
.../slider/common/params/ActionExistsArgs.java | 8 +-
.../slider/common/params/ActionLookupArgs.java | 76 ++++
.../slider/common/params/ActionThawArgs.java | 8 +
.../apache/slider/common/params/Arguments.java | 10 +-
.../apache/slider/common/params/ClientArgs.java | 21 +-
.../common/params/LaunchArgsAccessor.java | 4 +
.../common/params/LaunchArgsDelegate.java | 11 +
.../common/params/SliderAMCreateAction.java | 7 +
.../slider/common/params/SliderActions.java | 3 +
.../apache/slider/common/tools/SliderUtils.java | 128 +++++--
.../apache/slider/core/conf/MapOperations.java | 16 +-
.../slider/core/exceptions/ErrorStrings.java | 2 +-
.../slider/core/launch/AppMasterLauncher.java | 49 ++-
.../slider/core/launch/LaunchedApplication.java | 6 +
.../launch/SerializedApplicationReport.java | 92 +++++
.../core/persist/ApplicationReportSerDeser.java | 57 +++
.../slider/core/persist/JsonSerDeser.java | 25 +-
.../server/appmaster/SliderAppMaster.java | 46 ++-
.../server/appmaster/monkey/ChaosEntry.java | 9 +-
.../appmaster/monkey/ChaosMonkeyService.java | 47 ++-
.../TestFreezeThawFlexStandaloneAM.groovy | 52 ++-
.../slider/core/conf/TestConfTreeResolve.groovy | 22 +-
.../model/monkey/TestMockMonkey.groovy | 13 +-
.../slider/core/conf/examples/internal.json | 5 +-
.../framework/AgentCommandTestBase.groovy | 16 +-
.../funtest/framework/CommandTestBase.groovy | 344 ++++++++++++++++---
.../slider/funtest/framework/SliderShell.groovy | 5 +-
.../funtest/lifecycle/AMFailuresIT.groovy | 19 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 39 ++-
.../funtest/lifecycle/AgentFailures2IT.groovy | 24 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 22 +-
.../lifecycle/AgentLaunchFailureIT.groovy | 98 ++++++
.../funtest/lifecycle/AgentRegistryIT.groovy | 22 +-
.../lifecycle/AppsThroughAgentDemo.groovy | 167 +++++++++
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 35 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 28 +-
.../lifecycle/ClusterBuildDestroyIT.groovy | 1 -
43 files changed, 1448 insertions(+), 309 deletions(-)
----------------------------------------------------------------------
[24/50] git commit: SLIDER-570 sort found clusters by recent-ness
Posted by st...@apache.org.
SLIDER-570 sort found clusters by recent-ness
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7b630000
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7b630000
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7b630000
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 7b63000063609f9db0408e27b0b9356474f29ebc
Parents: 09d16a1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 29 11:44:37 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 21 +++++++++++---------
.../slider/client/SliderYarnClientImpl.java | 11 +++++-----
2 files changed, 17 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7b630000/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 392280a..6ebbcdd 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1693,13 +1693,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return actionList(clustername, args);
}
- /**
- * Implement the list action: list all nodes
-
- * @param clustername List out specific instance name
- * @param args Action list arguments
- * @return 0 if one or more entries were listed
- */
+ /**
+ * Implement the list action.
+ * @param clustername List out specific instance name
+ * @param args Action list arguments
+ * @return 0 if one or more entries were listed
+ * @throws IOException
+ * @throws YarnException
+ * @throws UnknownApplicationInstanceException if a specific instance
+ * was named but it was not found
+ */
@Override
@VisibleForTesting
public int actionList(String clustername, ActionListArgs args)
@@ -1996,8 +1999,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private RunningApplication findApplication(String appname)
throws YarnException, IOException {
ApplicationReport applicationReport = findInstance(appname);
- return applicationReport != null ? new RunningApplication(yarnClient, applicationReport): null;
-
+ return applicationReport != null ?
+ new RunningApplication(yarnClient, applicationReport): null;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7b630000/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 2f18b7a..a2a7fe7 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -259,19 +259,18 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
String appname) {
+ // sort by most recent
+ SliderUtils.sortApplicationsByMostRecent(instances);
ApplicationReport found = null;
- ApplicationReport foundAndLive = null;
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)) {
- found = app;
if (isApplicationLive(app)) {
- foundAndLive = app;
+ return app;
}
+ // set the found value if not set
+ found = found != null ? found : app;
}
}
- if (foundAndLive != null) {
- found = foundAndLive;
- }
return found;
}
[35/50] git commit: SLIDER-588 AppsThroughAgentIT test failed with
'Application registry is not accessible' ... more diagnostics now
Posted by st...@apache.org.
SLIDER-588 AppsThroughAgentIT test failed with 'Application registry is not accessible' ... more diagnostics now
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9e81c0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9e81c0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9e81c0b
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: a9e81c0b6ba05bc45d5c135c503238c12ea4e715
Parents: 2ac8428
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 17:06:48 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 17:06:48 2014 +0000
----------------------------------------------------------------------
.../apache/slider/funtest/framework/CommandTestBase.groovy | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9e81c0b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 072b07e..9a1824d 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -806,9 +806,14 @@ abstract class CommandTestBase extends SliderTestUtils {
PROBE_SLEEP_TIME,
[application: application],
true,
- 'Application registry is not accessible, failing test.') {
- describe "final state of app that tests say is not able to access registry"
+ "Application registry is not accessible after $REGISTRY_STARTUP_TIMEOUT") {
+ describe "Not able to access registry after after $REGISTRY_STARTUP_TIMEOUT"
exists(application, true).dumpOutput()
+ SliderShell shell = registry(0, [
+ ARG_NAME,
+ application,
+ ARG_LISTEXP
+ ])
}
}
[49/50] git commit: SLIDER-531 -keep in sync with latest YARN changes
Posted by st...@apache.org.
SLIDER-531 -keep in sync with latest YARN changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0161b590
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0161b590
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0161b590
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 0161b590ac4311512a5729f5e2f984d5d1dac319
Parents: ee92943
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 27 17:57:25 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Nov 2 14:14:08 2014 +0000
----------------------------------------------------------------------
.../server/appmaster/web/rest/registry/PathEntryMarshalling.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0161b590/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
index 937bebf..5f987c2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/registry/PathEntryMarshalling.groovy
@@ -23,6 +23,6 @@ import org.apache.hadoop.registry.client.binding.JsonSerDeser
class PathEntryMarshalling
extends JsonSerDeser<PathEntryResource> {
public PathEntryMarshalling() {
- super(PathEntryResource.class, new byte[0]);
+ super(PathEntryResource.class);
}
}
\ No newline at end of file
[14/50] git commit: Merge branch
'feature/SLIDER-581_funtest_return_values_on_windows' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-581_funtest_return_values_on_windows' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a19c5052
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a19c5052
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a19c5052
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: a19c50525e922214d0a52b1d99e9e90be51f7645
Parents: 05cf898 ef516d9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 14:38:29 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 14:38:29 2014 +0000
----------------------------------------------------------------------
.../slider/funtest/framework/SliderShell.groovy | 16 +++++---
.../funtest/commands/CommandExitCodesIT.groovy | 43 ++++++++++++++++++++
.../funtest/commands/ListCommandIT.groovy | 2 +-
.../commands/UnknownClusterOperationsIT.groovy | 2 -
4 files changed, 55 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[04/50] git commit: Adjust default memory requirements for HBase
Posted by st...@apache.org.
Adjust default memory requirements for HBase
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bbde42bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bbde42bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bbde42bd
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: bbde42bdf979e46e7198348eebcab724c21f9f1b
Parents: 74dfc5b
Author: tedyu <yu...@gmail.com>
Authored: Wed Oct 29 11:12:19 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Oct 29 11:12:19 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/resources-default.json | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bbde42bd/app-packages/hbase/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources-default.json b/app-packages/hbase/resources-default.json
index c467f1b..ff58989 100644
--- a/app-packages/hbase/resources-default.json
+++ b/app-packages/hbase/resources-default.json
@@ -10,29 +10,29 @@
"HBASE_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "1500"
},
"slider-appmaster": {
},
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "1500"
},
"HBASE_REST": {
"yarn.role.priority": "3",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "556"
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
"yarn.component.instances": "0",
- "yarn.memory": "256"
+ "yarn.memory": "556"
},
"HBASE_THRIFT2": {
"yarn.role.priority": "5",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "556"
}
}
}
[07/50] git commit: SLIDER-584 ensure that all of stdout/stderr gets
through the test run
Posted by st...@apache.org.
SLIDER-584 ensure that all of stdout/stderr gets through the test run
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/035dbe7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/035dbe7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/035dbe7f
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 035dbe7fec92baf97f97b45038f0882ff3306a44
Parents: 20b071e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 12:37:43 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 12:37:43 2014 +0000
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 30 +++++++++++++----
.../slider/core/main/ServiceLauncher.java | 2 ++
.../slider/funtest/framework/SliderShell.groovy | 35 +++++++++++---------
3 files changed, 44 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/035dbe7f/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index f3c3902..38ff68d 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -109,6 +109,19 @@ def info(text):
sys.stdout.flush()
+def out(toStdErr, text) :
+ """
+ Write to one of the system output channels.
+ This action does not add newlines. If you want that: write them yourself
+ :param toStdErr: flag set if stderr is to be the dest
+ :param text: text to write.
+ :return:
+ """
+ if toStdErr:
+ sys.stderr.write(text)
+ else:
+ sys.stdout.write(text)
+
def read(pipe, line):
"""
read a char, append to the listing if there is a char that is not \n
@@ -129,7 +142,7 @@ def read(pipe, line):
return line, False
-def print_output(name, src):
+def print_output(name, src, toStdErr):
"""
Relay the output stream to stdout line by line
:param name:
@@ -142,11 +155,14 @@ def print_output(name, src):
while not finished:
(line, done) = read(src, line)
if done:
- if DEBUG:
- info(name +': ' + line)
- else:
- info(line)
+ out(toStdErr, line + "\n")
line = ""
+ # closedown: read remainder of stream
+ c = src.read(1)
+ while c!="" :
+ out(toStdErr, c)
+ c = src.read(1)
+
src.close()
@@ -166,10 +182,10 @@ def runProcess(commandline):
bufsize=1,
close_fds=ON_POSIX)
- t = Thread(target=print_output, args=("stdout", exe.stdout))
+ t = Thread(target=print_output, args=("stdout", exe.stdout, False))
t.daemon = True
t.start()
- t2 = Thread(target=print_output, args=("stderr", exe.stderr,))
+ t2 = Thread(target=print_output, args=("stderr", exe.stderr, True))
t2.daemon = True
t2.start()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/035dbe7f/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
index 29b1d4e..498a8ad 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -401,6 +401,8 @@ public class ServiceLauncher<S extends Service>
Configuration conf = new Configuration();
String[] processedArgs = extractConfigurationArgs(conf, args);
ExitUtil.ExitException ee = launchServiceRobustly(conf, processedArgs);
+ System.out.flush();
+ System.err.flush();
exit(ee);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/035dbe7f/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index e278981..15bf339 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -228,7 +228,6 @@ class SliderShell extends Shell {
/**
* Execute shell script consisting of as many Strings as we have arguments,
- * possibly under an explicit username (requires sudoers privileges).
* NOTE: individual strings are concatenated into a single script as though
* they were delimited with new line character. All quoting rules are exactly
* what one would expect in standalone shell script.
@@ -251,21 +250,15 @@ class SliderShell extends Shell {
writer.close()
}
ByteArrayOutputStream baosErr = new ByteArrayOutputStream(4096);
- proc.consumeProcessErrorStream(baosErr);
- out = proc.in.readLines()
-
- // Possibly a bug in String.split as it generates a 1-element array on an
- // empty String
-
- if (baosErr.size() != 0) {
- err = baosErr.toString().split('\n');
- } else {
- err = [];
- }
+ ByteArrayOutputStream baosOut = new ByteArrayOutputStream(4096);
+ proc.consumeProcessOutput(baosOut, baosErr)
proc.waitFor()
ret = proc.exitValue()
+ out = streamToLines(baosOut)
+ err = streamToLines(baosErr)
+
if (LOG.isTraceEnabled()) {
if (ret != 0) {
LOG.trace("return: $ret");
@@ -274,15 +267,25 @@ class SliderShell extends Shell {
LOG.trace("\n<stdout>\n${out.join('\n')}\n</stdout>");
}
- def stderror = super.err
- if (stderror.size() != 0) {
- LOG.trace("\n<stderr>\n${stderror.join('\n')}\n</stderr>");
+ if (err.size() != 0) {
+ LOG.trace("\n<stderr>\n${err.join('\n')}\n</stderr>");
}
}
-
return this
}
+ /**
+ * Convert a stream to lines in an array
+ * @param out output stream
+ * @return the list of entries
+ */
+ protected List<String> streamToLines(ByteArrayOutputStream out) {
+ if (out.size() != 0) {
+ return out.toString().split('\n');
+ } else {
+ return [];
+ }
+ }
public String findLineEntry(String[] locaters) {
int index = 0;
[32/50] git commit: SLIDER-570: AgentClusterLifecycleIT ->
lookup/appId driven on thaw operations
Posted by st...@apache.org.
SLIDER-570: AgentClusterLifecycleIT -> lookup/appId driven on thaw operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1f98074f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1f98074f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1f98074f
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 1f98074f16e7840d11727d9bcf1efb5a7858595e
Parents: e1023ea
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 14:24:44 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 14:24:44 2014 +0000
----------------------------------------------------------------------
.../funtest/framework/CommandTestBase.groovy | 1 +
.../lifecycle/AgentClusterLifecycleIT.groovy | 26 +++++++++++++++++---
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f98074f/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 7ea84df..072b07e 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -692,6 +692,7 @@ abstract class CommandTestBase extends SliderTestUtils {
"launch",
".json",
new File("target"))
+ reportFile.delete()
return reportFile
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f98074f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index cb137ce..a0cb6ca 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
@@ -65,6 +66,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
+
File launchReportFile = createAppReportFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
@@ -138,6 +140,9 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
])
describe " >>> Cluster is now frozen."
+
+ // should be in finished state, as this was a clean shutdown
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
//cluster exists if you don't want it to be live
exists(0, CLUSTER, false)
@@ -152,12 +157,19 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
list(-1, [ARG_LIVE])
list(-1, [ARG_STATE, "running"])
list( 0, [ARG_STATE, "FINISHED"])
-
+
+ def thawReport = createAppReportFile()
//start then stop the cluster
thaw(CLUSTER,
[
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ARG_OUTPUT, thawReport.absolutePath,
])
+ def thawedAppId = ensureYarnApplicationIsUp(thawReport)
+
+
+ assertAppRunning(thawedAppId)
+
exists(0, CLUSTER)
describe " >>> Cluster is now thawed."
list(0, [CLUSTER, ARG_LIVE])
@@ -175,6 +187,8 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
describe " >>> Cluster is now force frozen - 2nd time."
+ // new instance should be in killed state
+ assertInYarnState(thawedAppId, YarnApplicationState.KILLED)
//cluster is no longer live
exists(0, CLUSTER, false)
@@ -183,19 +197,23 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
//start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
-
+
+ def thawReport2 = createAppReportFile()
+ //start then stop the cluster
thaw(CLUSTER,
[
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
- ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3"
+ ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3",
+ ARG_OUTPUT, thawReport2.absolutePath
])
-
+ def thawedAppId2 = ensureYarnApplicationIsUp(thawReport2)
describe " >>> Cluster is now thawed - 2nd time."
describe " >>> Kill AM and wait for restart."
ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
+ assertAppRunning(thawedAppId2)
def restarted = status.getInfo(
StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null
[25/50] git commit: SLIDER-572 add startup delay for the chaos monkey
Posted by st...@apache.org.
SLIDER-572 add startup delay for the chaos monkey
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/517042f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/517042f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/517042f2
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 517042f2e6878445610884eaa1b47fa3a349ed17
Parents: 25bbfeb
Author: Steve Loughran <st...@apache.org>
Authored: Wed Oct 29 20:46:32 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/api/InternalKeys.java | 11 ++++++++-
.../apache/slider/core/conf/MapOperations.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 24 ++++++++++----------
.../appmaster/monkey/ChaosMonkeyService.java | 20 +++++++++-------
.../model/monkey/TestMockMonkey.groovy | 4 ++--
5 files changed, 37 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index 5f150e6..074644d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -121,6 +121,14 @@ public interface InternalKeys {
int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
+ String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay";
+ String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days";
+ String CHAOS_MONKEY_DELAY_HOURS = CHAOS_MONKEY_DELAY + ".hours";
+ String CHAOS_MONKEY_DELAY_MINUTES = CHAOS_MONKEY_DELAY + ".minutes";
+ String CHAOS_MONKEY_DELAY_SECONDS = CHAOS_MONKEY_DELAY + ".seconds";
+
+ int DEFAULT_CHAOS_MONKEY_STARTUP_DELAY = 0;
+
/**
* Prefix for all chaos monkey probabilities
*/
@@ -133,7 +141,8 @@ public interface InternalKeys {
/**
* Probability of a monkey check killing the AM: {@value}
*/
- String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = CHAOS_MONKEY_PROBABILITY +".amfailure";
+ String CHAOS_MONKEY_PROBABILITY_AM_FAILURE =
+ CHAOS_MONKEY_PROBABILITY + ".amfailure";
/**
* Default probability of a monkey check killing the AM: {@value}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 6503c9b..de8fc2c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -282,7 +282,7 @@ public class MapOperations implements Map<String, String> {
* @param defHours
* @param defMins
* @param defSecs
- * @return
+ * @return the aggregate time range in seconds
*/
public long getTimeRange(String basekey,
int defDays,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 39a2572..855ecd7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1101,14 +1101,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
-/*
-
- @Override
- protected RegistryOperationsService createRegistryOperationsInstance() {
- return new ResourceManagerRegistryService("YarnRegistry");
- }
-*/
-
/**
* TODO: purge this once RM is doing the work
* @throws IOException
@@ -1133,7 +1125,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
if (instance == null) {
return false;
}
- // this is where component registrations will go
+ // this is where component registrations go
log.info("Registering component {}", id);
String cid = RegistryPathUtils.encodeYarnID(id.toString());
ServiceRecord container = new ServiceRecord();
@@ -2185,8 +2177,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
"Chaos monkey not configured with a time interval...not enabling");
return false;
}
- log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
- monkeyInterval, monkeyInterval/(60*60));
+
+ long monkeyDelay = internals.getTimeRange(
+ InternalKeys.CHAOS_MONKEY_DELAY,
+ 0,
+ 0,
+ 0,
+ (int)monkeyInterval);
+
+ log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours -delay {}",
+ monkeyInterval, monkeyInterval/(60*60), monkeyDelay);
monkey = new ChaosMonkeyService(metrics, actionQueues);
initAndAddService(monkey);
@@ -2204,7 +2204,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
containerKillProbability);
// and schedule it
- if (monkey.schedule(monkeyInterval, TimeUnit.SECONDS)) {
+ if (monkey.schedule(monkeyDelay, monkeyInterval, TimeUnit.SECONDS)) {
log.info("Chaos Monkey is running");
return true;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index f7b1eb7..80f981c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -92,13 +92,14 @@ public class ChaosMonkeyService extends AbstractService {
/**
* Schedule the monkey
- * @param time interval
+ *
+ * @param delay initial delay
* @param timeUnit time unit
* @return true if it was scheduled (i.e. 1+ action) and interval > 0
*/
- public boolean schedule(long time, TimeUnit timeUnit) {
- if (time > 0 && !chaosEntries.isEmpty()) {
- queues.schedule(getChaosAction(time, timeUnit));
+ public boolean schedule(long delay, long interval, TimeUnit timeUnit) {
+ if (interval > 0 && !chaosEntries.isEmpty()) {
+ queues.schedule(getChaosAction(delay, interval, timeUnit));
return true;
} else {
return false;
@@ -107,15 +108,18 @@ public class ChaosMonkeyService extends AbstractService {
/**
* Get the chaos action
- * @param time interval
+ *
+ * @param delay
* @param timeUnit time unit
* @return the action to schedule
*/
- public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
+ public RenewingAction<MonkeyPlayAction> getChaosAction(long delay,
+ long interval,
+ TimeUnit timeUnit) {
RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>(
new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
- time,
- time,
+ delay,
+ interval,
timeUnit,
0
);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/517042f2/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
index 26d07ee..d31c9f6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -78,7 +78,7 @@ class TestMockMonkey extends BaseMockAppStateTest {
assert 0 == monkey.targetCount;
monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
assert 1 == monkey.targetCount;
- assert monkey.schedule(1, TimeUnit.SECONDS)
+ assert monkey.schedule(0, 1, TimeUnit.SECONDS)
assert 1 == queues.scheduledActions.size()
}
@@ -96,7 +96,7 @@ class TestMockMonkey extends BaseMockAppStateTest {
public void testMonkeyScheduleProb0Actions() throws Throwable {
ChaosCounter counter = new ChaosCounter()
monkey.addTarget("target", counter, 0)
- assert !monkey.schedule(1, TimeUnit.SECONDS)
+ assert !monkey.schedule(0, 1, TimeUnit.SECONDS)
assert 0 == queues.scheduledActions.size()
}
[17/50] git commit: SLIDER-591 regression: ListCommandIT broken with
changed exit codes
Posted by st...@apache.org.
SLIDER-591 regression: ListCommandIT broken with changed exit codes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fdcc5c2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fdcc5c2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fdcc5c2c
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: fdcc5c2cfaa60d4a80506dc9e141e824a5fdc1b2
Parents: e771352
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 23:09:11 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 23:09:11 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/funtest/commands/ListCommandIT.groovy | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fdcc5c2c/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
index c65280a..916117c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.commands
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.BeforeClass
import org.junit.Test
@@ -36,13 +37,7 @@ public class ListCommandIT extends CommandTestBase {
@Test
public void testListAllLive() throws Throwable {
def shell = list("--live")
- assert shell.ret == 0 || shell.ret == 1
- }
-
- @Test
- public void testListAllFinishedLive() throws Throwable {
- def shell = list("--live")
- assert shell.ret == 0 || shell.ret == 1
+ assert shell.ret == 0 || shell.ret == LauncherExitCodes.EXIT_FALSE
}
}
[05/50] git commit: SLIDER-560 add new accumulo appConfig to assembly
Posted by st...@apache.org.
SLIDER-560 add new accumulo appConfig to assembly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8c2e09f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8c2e09f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8c2e09f1
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 8c2e09f15dee00bf494bbe089f94e8f291576f99
Parents: bbde42b
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Oct 29 14:01:54 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Oct 29 14:01:54 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/src/assembly/accumulo.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8c2e09f1/app-packages/accumulo/src/assembly/accumulo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/assembly/accumulo.xml b/app-packages/accumulo/src/assembly/accumulo.xml
index 2a27638..7be1942 100644
--- a/app-packages/accumulo/src/assembly/accumulo.xml
+++ b/app-packages/accumulo/src/assembly/accumulo.xml
@@ -35,6 +35,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>appConfig-secured-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>metainfo.xml</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
@@ -64,6 +70,7 @@
<exclude>src/**</exclude>
<exclude>target/**</exclude>
<exclude>appConfig-default.json</exclude>
+ <exclude>appConfig-secured-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
[02/50] git commit: SLIDER-576. Default packages have a hard-coded
build version - instead they need to use mvn parameter
Posted by st...@apache.org.
SLIDER-576. Default packages have a hard-coded build version - instead they need to use mvn parameter
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6ae705ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6ae705ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6ae705ac
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 6ae705ac6f7597609958612ec34899f8e3ad33c1
Parents: 196466d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Oct 29 09:43:28 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Oct 29 09:43:28 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/src/assembly/hbase.xml | 7 +++++++
app-packages/storm/src/assembly/storm.xml | 7 +++++++
2 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6ae705ac/app-packages/hbase/src/assembly/hbase.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/src/assembly/hbase.xml b/app-packages/hbase/src/assembly/hbase.xml
index a94c827..a74304c 100644
--- a/app-packages/hbase/src/assembly/hbase.xml
+++ b/app-packages/hbase/src/assembly/hbase.xml
@@ -36,6 +36,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>appConfig-secured-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>metainfo.xml</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
@@ -58,6 +64,7 @@
<exclude>src/**</exclude>
<exclude>target/**</exclude>
<exclude>appConfig-default.json</exclude>
+ <exclude>appConfig-secured-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6ae705ac/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
index cc000c5..f7dcf13 100644
--- a/app-packages/storm/src/assembly/storm.xml
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -36,6 +36,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>appConfig-secured-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>metainfo.xml</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
@@ -58,6 +64,7 @@
<exclude>src/**</exclude>
<exclude>target/**</exclude>
<exclude>appConfig-default.json</exclude>
+ <exclude>appConfig-secured-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
[30/50] git commit: SLIDER-570 migrate tests to app-id based startup
cycle
Posted by st...@apache.org.
SLIDER-570 migrate tests to app-id based startup cycle
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7df654f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7df654f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7df654f
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: e7df654f29e414b822a93738321940dc8f164114
Parents: 1cd37f5
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 22:40:14 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:50 2014 +0000
----------------------------------------------------------------------
.../apache/slider/core/conf/MapOperations.java | 14 +-
.../server/appmaster/SliderAppMaster.java | 2 +
.../slider/core/conf/TestConfTreeResolve.groovy | 22 ++-
.../slider/core/conf/examples/internal.json | 5 +-
.../funtest/framework/CommandTestBase.groovy | 193 ++++++++++++++-----
.../funtest/lifecycle/AMFailuresIT.groovy | 16 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 2 +-
.../funtest/lifecycle/AgentFailures2IT.groovy | 16 +-
.../funtest/lifecycle/AgentFailuresIT.groovy | 16 +-
.../lifecycle/AgentLaunchFailureIT.groovy | 34 ++--
.../funtest/lifecycle/AgentRegistryIT.groovy | 16 +-
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 15 +-
.../AppsThroughAgentQueueAndLabelsIT.groovy | 22 ++-
13 files changed, 259 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index de8fc2c..5f7b5f0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -38,6 +38,10 @@ import java.util.Set;
public class MapOperations implements Map<String, String> {
private static final Logger log =
LoggerFactory.getLogger(MapOperations.class);
+ public static final String DAYS = ".days";
+ public static final String HOURS = ".hours";
+ public static final String MINUTES = ".minutes";
+ public static final String SECONDS = ".seconds";
/**
* Global options
@@ -277,7 +281,7 @@ public class MapOperations implements Map<String, String> {
/**
* Get the time range of a set of keys
- * @param basekey
+ * @param basekey base key to which suffix gets applied
* @param defDays
* @param defHours
* @param defMins
@@ -290,11 +294,11 @@ public class MapOperations implements Map<String, String> {
int defMins,
int defSecs) {
Preconditions.checkArgument(basekey != null);
- int days = getOptionInt(basekey + ".days", defDays);
- int hours = getOptionInt(basekey + ".hours", defHours);
+ int days = getOptionInt(basekey + DAYS, defDays);
+ int hours = getOptionInt(basekey + HOURS, defHours);
- int minutes = getOptionInt(basekey + ".minutes", defMins);
- int seconds = getOptionInt(basekey + ".seconds", defSecs);
+ int minutes = getOptionInt(basekey + MINUTES, defMins);
+ int seconds = getOptionInt(basekey + SECONDS, defSecs);
// range check
Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
&& seconds >= 0,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5e2ba11..f2ea00f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -2199,12 +2199,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
0);
if (amLaunchFailProbability> 0 && monkey.chaosCheck(amLaunchFailProbability)) {
+ log.info("Chaos Monkey has triggered AM Launch failure");
// trigger a failure
ActionStopSlider stop = new ActionStopSlider("stop",
0, TimeUnit.SECONDS,
LauncherExitCodes.EXIT_FALSE,
FinalApplicationStatus.FAILED,
E_TRIGGERED_LAUNCH_FAILURE);
+ queue(stop);
}
int amKillProbability = internals.getOptionInt(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
index b655be8..156ae71 100644
--- a/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/core/conf/TestConfTreeResolve.groovy
@@ -19,10 +19,11 @@
package org.apache.slider.core.conf
import groovy.util.logging.Slf4j
+import static org.apache.slider.api.InternalKeys.*
import org.junit.Assert
import org.junit.Test
-import static org.apache.slider.core.conf.ExampleConfResources.overridden
+import static org.apache.slider.core.conf.ExampleConfResources.*
/**
* Test
@@ -89,4 +90,23 @@ class TestConfTreeResolve extends Assert {
assert worker["timeout"] == "1000"
}
+
+ @Test
+ public void testTimeIntervalLoading() throws Throwable {
+
+ def orig = ExampleConfResources.loadResource(internal)
+
+ MapOperations internals = new MapOperations(orig.global)
+ def s = internals.getOptionInt(
+ CHAOS_MONKEY_INTERVAL + MapOperations.SECONDS,
+ 0)
+ assert s == 60
+ long monkeyInterval = internals.getTimeRange(
+ CHAOS_MONKEY_INTERVAL,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
+ DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
+ 0);
+ assert monkeyInterval == 60;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
index 8617d1f..4c782fb 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
@@ -7,7 +7,10 @@
"global": {
"application.name": "small_cluster",
"application.type": "hbase",
- "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar",
+ "internal.chaos.monkey.probability.amlaunchfailure": "10000",
+ "internal.chaos.monkey.interval.seconds": "60",
+ "internal.chaos.monkey.enabled": "true"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 7c479af..0a7b295 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.common.tools.Duration
import org.apache.slider.core.launch.SerializedApplicationReport
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
@@ -90,7 +91,12 @@ abstract class CommandTestBase extends SliderTestUtils {
* not need to be escaped
*/
public static final String TILDE
-
+ public static final int CONTAINER_LAUNCH_TIMEOUT = 90000
+ public static final int PROBE_SLEEP_TIME = 4000
+ public static final int REGISTRY_STARTUP_TIMEOUT = 60000
+ public static
+ final String E_LAUNCH_FAIL = 'Application did not start'
+
/*
Static initializer for test configurations. If this code throws exceptions
(which it may) the class will not be instantiable.
@@ -710,6 +716,16 @@ abstract class CommandTestBase extends SliderTestUtils {
}
return null;
}
+
+ public static SerializedApplicationReport loadAppReport(File reportFile) {
+ if (reportFile.exists() && reportFile.length()> 0) {
+ ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
+ def report = serDeser.fromFile(reportFile)
+ return report
+ } else {
+ throw new FileNotFoundException(reportFile.absolutePath)
+ }
+ }
public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
def report = maybeLoadAppReport(launchReport)
@@ -776,9 +792,9 @@ abstract class CommandTestBase extends SliderTestUtils {
}
protected void ensureRegistryCallSucceeds(String application) {
- repeatUntilTrue(this.&isRegistryAccessible,
- 10,
- 5 * 1000,
+ repeatUntilSuccess(this.&isRegistryAccessible,
+ REGISTRY_STARTUP_TIMEOUT,
+ PROBE_SLEEP_TIME,
[application: application],
true,
'Application registry is not accessible, failing test.') {
@@ -789,35 +805,36 @@ abstract class CommandTestBase extends SliderTestUtils {
protected void ensureApplicationIsUp(String application) {
- repeatUntilTrue(this.&isApplicationRunning,
- 30,
+ repeatUntilSuccess(this.&isApplicationRunning,
SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
- DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
+ DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS) * 1000,
+ PROBE_SLEEP_TIME,
[application: application],
true,
- 'Application did not start, failing test.') {
+ E_LAUNCH_FAIL) {
describe "final state of app that tests say is not up"
exists(application, true).dumpOutput()
}
}
- protected boolean isRegistryAccessible(Map<String, String> args) {
+ protected Outcome isRegistryAccessible(Map<String, String> args) {
String applicationName = args['application'];
SliderShell shell = slider(
[
ACTION_REGISTRY,
ARG_NAME,
applicationName,
- ARG_LISTEXP])
+ ARG_LISTEXP
+ ])
if (EXIT_SUCCESS != shell.execute()) {
logShell(shell)
}
- return EXIT_SUCCESS == shell.execute()
+ return Outcome.fromBool(EXIT_SUCCESS == shell.execute())
}
- protected boolean isApplicationRunning(Map<String, String> args) {
+ protected Outcome isApplicationRunning(Map<String, String> args) {
String applicationName = args['application'];
- return isApplicationUp(applicationName);
+ return Outcome.fromBool(isApplicationUp(applicationName))
}
protected boolean isApplicationUp(String applicationName) {
@@ -827,18 +844,6 @@ abstract class CommandTestBase extends SliderTestUtils {
);
}
- protected void ensureYarnApplicationIsUp(String application) {
- repeatUntilTrue(this.&isApplicationRunning,
- 30,
- SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
- DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
- [application: application],
- true,
- 'Application did not start, failing test.') {
- describe "final state of app that tests say is not up"
- exists(application, true).dumpOutput()
- }
- }
/**
* is an application in a desired yarn state
@@ -853,27 +858,111 @@ abstract class CommandTestBase extends SliderTestUtils {
[ACTION_EXISTS, applicationName, ARG_STATE, yarnState.toString()])
return shell.ret == 0
}
-
+
+
+ protected Outcome isYarnApplicationRunning(Map<String, String> args) {
+ String applicationId = args['applicationId'];
+ return isYarnApplicationRunning(applicationId)
+ }
+
/**
* is a yarn application in a desired yarn state
* @param yarnState
* @param applicationName
- * @return
+ * @return an outcome indicating whether the app is at the state, on its way
+ * or has gone past
*/
- public static boolean isYarnApplicationInState(
- String applicationId,
- YarnApplicationState yarnState) {
+ public static Outcome isYarnApplicationRunning(
+ String applicationId) {
+ YarnApplicationState appState = lookupYarnAppState(applicationId)
+ YarnApplicationState yarnState = YarnApplicationState.RUNNING
+ if (yarnState == appState) {
+ return Outcome.Success;
+ }
+
+ if (appState.ordinal() > yarnState.ordinal()) {
+ // app has passed beyond hope
+ return Outcome.Fail
+ }
+ return Outcome.Retry
+ }
+
+ public static YarnApplicationState lookupYarnAppState(String applicationId) {
def sar = lookupApplication(applicationId)
assert sar != null;
- return yarnState.toString() == sar.state
+ YarnApplicationState appState = YarnApplicationState.valueOf(sar.state)
+ return appState
+ }
+
+ public static void assertInYarnState(String applicationId,
+ YarnApplicationState expectedState) {
+ def applicationReport = lookupApplication(applicationId)
+ assert expectedState.toString() == applicationReport.state
}
+ /**
+ * Wait for the YARN app to come up. This will fail fast
+ * @param launchReportFile launch time file containing app id
+ * @return the app ID
+ */
+ protected String ensureYarnApplicationIsUp(File launchReportFile) {
+ def id = loadAppReport(launchReportFile).applicationId
+ ensureYarnApplicationIsUp(id)
+ return id;
+ }
+ /**
+ * Wait for the YARN app to come up. This will fail fast
+ * @param applicationId
+ */
+ protected void ensureYarnApplicationIsUp(String applicationId) {
+ repeatUntilSuccess(this.&isYarnApplicationRunning,
+ SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
+ DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
+ PROBE_SLEEP_TIME,
+ [applicationId: applicationId],
+ true,
+ E_LAUNCH_FAIL) {
+ describe "final state of app that tests say is not up"
+ def sar = lookupApplication(applicationId)
+
+ def message = E_LAUNCH_FAIL + "\n$sar"
+ log.error(message)
+ fail(message)
+ }
+ }
+
+ /**
+ * Outcome for probes
+ */
+ static class Outcome {
+
+ public final String name;
+
+ private Outcome(String name) {
+ this.name = name
+ }
+
+ static Outcome Success = new Outcome("Success")
+ static Outcome Retry = new Outcome("Retry")
+ static Outcome Fail = new Outcome("Fail")
+
+
+ /**
+ * build from a bool, where false is mapped to retry
+ * @param b boolean
+ * @return an outcome
+ */
+ static Outcome fromBool(boolean b) {
+ return b? Success: Retry;
+ }
+
+ }
/**
* Repeat a probe until it succeeds, if it does not execute a failure
* closure then raise an exception with the supplied message
* @param probe probe
- * @param maxAttempts max number of attempts
+ * @param timeout time in millis before giving up
* @param sleepDur sleep between failing attempts
* @param args map of arguments to the probe
* @param failIfUnsuccessful if the probe fails after all the attempts
@@ -881,23 +970,35 @@ abstract class CommandTestBase extends SliderTestUtils {
* @param failureMessage message to include in exception raised
* @param failureHandler closure to invoke prior to the failure being raised
*/
- protected void repeatUntilTrue(Closure probe,
- int maxAttempts, int sleepDur, Map args,
- boolean failIfUnsuccessful = false,
+ protected void repeatUntilSuccess(Closure probe,
+ int timeout, int sleepDur,
+ Map args,
+ boolean failIfUnsuccessful,
String failureMessage,
Closure failureHandler) {
int attemptCount = 0
boolean succeeded = false;
- while (attemptCount < maxAttempts) {
- if (probe(args)) {
- // finished
+ boolean completed = false;
+ Duration duration = new Duration(timeout)
+ duration.start();
+ while (!completed) {
+ Outcome outcome = (Outcome) probe(args)
+ if (outcome.equals(Outcome.Success)) {
+ // success
log.debug("Success after $attemptCount attempt(s)")
succeeded = true;
- break
- };
- attemptCount++;
-
- sleep(sleepDur)
+ completed = true;
+ } else if (outcome.equals(Outcome.Retry)) {
+ // failed but retry possible
+ attemptCount++;
+ completed = duration.limitExceeded
+ if (!completed) {
+ sleep(sleepDur)
+ }
+ } else if (outcome.equals(Outcome.Fail)) {
+ // fast fail
+ completed = true;
+ }
}
if (failIfUnsuccessful & !succeeded) {
@@ -962,10 +1063,10 @@ abstract class CommandTestBase extends SliderTestUtils {
void expectContainerRequestedCountReached(String application, String role, int limit) {
- repeatUntilTrue(
+ repeatUntilSuccess(
this.&hasRequestedContainerCountReached,
- 90,
- 1000,
+ CONTAINER_LAUNCH_TIMEOUT,
+ PROBE_SLEEP_TIME,
[limit : Integer.toString(limit),
role : role,
application: application],
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 750a3d4..2e28c84 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -61,12 +61,15 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Test
public void testAMKilledWithStateAMStartedAgentsStarted() throws Throwable {
cleanup(APPLICATION_NAME)
+ File launchReportFile = createAppReportFile();
+
SliderShell shell = createTemplatedSliderApplication(
- APPLICATION_NAME, APP_TEMPLATE, APP_RESOURCE
- )
+ APPLICATION_NAME, APP_TEMPLATE, APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1)
// Wait for 20 secs for AM and agent to both reach STARTED state
@@ -92,13 +95,12 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
killAMUsingVagrantShell()
// Check that the application is not running (and is in ACCEPTED state)
- assert isApplicationInState(APPLICATION_NAME, YarnApplicationState.ACCEPTED
- ),
+ assert lookupYarnAppState(appId) == YarnApplicationState.ACCEPTED ,
'App should be in ACCEPTED state (since AM got killed)'
log.info("After AM KILL: application {} is in ACCEPTED state", APPLICATION_NAME)
// Wait until AM comes back up and verify container count again
- ensureApplicationIsUp(APPLICATION_NAME)
+ ensureYarnApplicationIsUp(appId)
// There should be exactly 1 live logger container
def cd2 = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
@@ -107,7 +109,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
def loggerStats2 = cd2.statistics[COMMAND_LOGGER]
assert origRequested == loggerStats2["containers.requested"],
'No new agent containers should be requested'
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ assert lookupYarnAppState(appId) == YarnApplicationState.RUNNING
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 3783a37..3e5cec7 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -78,7 +78,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
def launchReport = maybeLoadAppReport(launchReportFile)
assert launchReport;
- ensureApplicationIsUp(CLUSTER)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
//at this point the cluster should exist.
assertPathExists(clusterFS, "Cluster parent directory does not exist", clusterpath.parent)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 9b35fa4..6c6b52b 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -50,21 +51,20 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
- SliderShell shell = createTemplatedSliderApplication(
- APPLICATION_NAME,
- APP_TEMPLATE3,
- APP_RESOURCE)
+ File launchReportFile = createAppReportFile();
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME, APP_TEMPLATE3, APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
sleep(1000 * 20)
def cd = execStatus(APPLICATION_NAME)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
-
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
-
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 3847e3f..d5be0f8 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -20,11 +20,13 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
import org.junit.Test
@@ -50,20 +52,24 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
cleanup(APPLICATION_NAME)
- def shell = createTemplatedSliderApplication(APPLICATION_NAME,
- APP_TEMPLATE2,
- APP_RESOURCE)
+ File launchReportFile = createAppReportFile();
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME,
+ APP_TEMPLATE2,
+ APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2)
sleep(1000 * 20)
assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
index 4dcbed0..5087c06 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -20,6 +20,8 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.SliderAppMaster
+
import static org.apache.slider.api.InternalKeys.*
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
@@ -65,33 +67,33 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
APP_RESOURCE2,
[
ARG_OPTION, CHAOS_MONKEY_ENABLED, "true",
+ ARG_OPTION, CHAOS_MONKEY_INTERVAL_SECONDS, "60",
ARG_OPTION, CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE,
Integer.toString(PROBABILITY_PERCENT_100),
],
launchReportFile)
- shell.dumpOutput();
assert launchReportFile.exists()
assert launchReportFile.size() > 0
def launchReport = maybeLoadAppReport(launchReportFile)
assert launchReport;
assert launchReport.applicationId;
- def report = maybeLookupFromLaunchReport(launchReportFile)
- assert report;
- ensureApplicationIsUp(CLUSTER)
-
- //stop
- freeze(0, CLUSTER,
- [
- ARG_FORCE,
- ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
- ARG_MESSAGE, "final-shutdown"
- ])
-
- destroy(0, CLUSTER)
- //cluster now missing
- exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+ // spin expecting failure
+ def appId = launchReport.applicationId
+ sleep(5000)
+ describe("Awaiting failure")
+ try {
+ ensureYarnApplicationIsUp(appId)
+ fail("application is up")
+ } catch (AssertionError e) {
+ if(!e.toString().contains(SliderAppMaster.E_TRIGGERED_LAUNCH_FAILURE)) {
+ throw e;
+ }
+ }
+ def sar = lookupApplication(appId)
+ log.info(sar.toString())
+ assert sar.diagnostics.contains(SliderAppMaster.E_TRIGGERED_LAUNCH_FAILURE)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index 16e65fa..b5ee23d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.binding.RegistryUtils
import org.apache.hadoop.registry.client.types.Endpoint
import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
@@ -59,19 +60,17 @@ public class AgentRegistryIT extends AgentCommandTestBase
@Test
public void testAgentRegistry() throws Throwable {
describe("Create a 0-role cluster and make registry queries against it")
-
- // sanity check to verify the config is correct
- assert clusterFS.uri.scheme != "file"
-
def clusterpath = buildClusterPath(CLUSTER)
- assert !clusterFS.exists(clusterpath)
+ File launchReportFile = createAppReportFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
- APP_RESOURCE2)
+ APP_RESOURCE2,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(CLUSTER)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
//at this point the cluster should exist.
assertPathExists(
@@ -124,15 +123,16 @@ public class AgentRegistryIT extends AgentCommandTestBase
//stop
freeze(0, CLUSTER,
[
- ARG_FORCE,
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "final-shutdown"
])
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
destroy(0, CLUSTER)
//cluster now missing
exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 488173b..5a5b964 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -53,13 +54,15 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
+ File launchReportFile = createAppReportFile();
SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
APP_TEMPLATE,
- APP_RESOURCE)
-
+ APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
//flex
slider(EXIT_SUCCESS,
@@ -107,8 +110,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
APPLICATION_NAME,
ARG_GETEXP,
"container_log_dirs"])
- if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
- || !containsString(shell, "\"level\" : \"component\"", 2)) {
+ if (!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
+ || !containsString(shell, "\"level\" : \"component\"", 2)) {
logShell(shell)
assert fail("Should list 2 entries for log folders")
}
@@ -159,6 +162,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assert fail("Should have exported cl-site")
}
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7df654f/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index ec999f5..ee418dc 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -80,15 +81,16 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeLabelsRedAndBlueAdded()
cleanup(APPLICATION_NAME)
- SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
+ File launchReportFile = createAppReportFile();
+ SliderShell shell = createTemplatedSliderApplication(
+ APPLICATION_NAME,
APP_TEMPLATE,
APP_RESOURCE4,
- [ARG_QUEUE, TARGET_QUEUE]
- )
-
+ [ARG_QUEUE, TARGET_QUEUE],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1 )
expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
@@ -100,18 +102,18 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
APPLICATION_NAME,
ARG_COMPONENT,
COMMAND_LOGGER,
- "3"])
+ "3"
+ ])
- // sleep till the new instance starts
- ensureApplicationIsUp(APPLICATION_NAME)
+ // spin till the flexed instance starts
+ ensureYarnApplicationIsUp(appId)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
sleep(1000 * 20)
def cd = execStatus(APPLICATION_NAME)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
-
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
}
[48/50] git commit: SLIDER-531. YARN-2678 new registry formats
Posted by st...@apache.org.
SLIDER-531. YARN-2678 new registry formats
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ee929430
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ee929430
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ee929430
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: ee9294302551c4cb4f52c41f67f8fc17b357787a
Parents: 96c9006
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 23 16:00:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Nov 2 14:14:08 2014 +0000
----------------------------------------------------------------------
slider-agent/src/main/python/agent/Registry.py | 11 ++++++-----
.../apache/slider/server/appmaster/SliderAppMaster.java | 3 +--
.../slider/funtest/lifecycle/AgentRegistryIT.groovy | 2 +-
3 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee929430/slider-agent/src/main/python/agent/Registry.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Registry.py b/slider-agent/src/main/python/agent/Registry.py
index ad84053..6d34956 100644
--- a/slider-agent/src/main/python/agent/Registry.py
+++ b/slider-agent/src/main/python/agent/Registry.py
@@ -24,7 +24,7 @@ from kazoo.client import KazooClient
logger = logging.getLogger()
-class Registry:
+class Registry(object):
def __init__(self, zk_quorum, zk_reg_path):
self.zk_quorum = zk_quorum
self.zk_reg_path = zk_reg_path
@@ -39,17 +39,18 @@ class Registry:
zk = KazooClient(hosts=self.zk_quorum, read_only=True)
zk.start()
data, stat = zk.get(self.zk_reg_path)
- data = data.lstrip("jsonservicerec")
logger.debug("Registry Data: %s" % (data.decode("utf-8")))
sliderRegistry = json.loads(data)
internalAttr = sliderRegistry["internal"]
for internal in internalAttr:
if internal["api"] == "org.apache.slider.agents.secure":
- amUrl = internal["addresses"][0][0]
+ address0 = internal["addresses"][0]
+ amUrl = address0["uri"]
amHost = amUrl.split("/")[2].split(":")[0]
amSecuredPort = amUrl.split(":")[2].split("/")[0]
if internal["api"] == "org.apache.slider.agents.oneway":
- amUnsecureUrl = internal["addresses"][0][0]
+ address0 = internal["addresses"][0]
+ amUnsecureUrl = address0["uri"]
amHost = amUnsecureUrl.split("/")[2].split(":")[0]
amUnsecuredPort = amUnsecureUrl.split(":")[2].split("/")[0]
@@ -62,7 +63,7 @@ class Registry:
(self.zk_reg_path, self.zk_quorum, str(e)))
pass
finally:
- if not zk == None:
+ if not zk is None:
zk.stop()
zk.close()
logger.info("AM Host = %s, AM Secured Port = %s, ping port = %s" % (amHost, amSecuredPort, amUnsecuredPort))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee929430/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index f2ea00f..3bee4a4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1060,8 +1060,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceRecord.addExternalEndpoint(
RegistryTypeUtils.ipcEndpoint(
CustomRegistryConstants.AM_IPC_PROTOCOL,
- true,
- RegistryTypeUtils.marshall(rpcServiceAddress)));
+ rpcServiceAddress));
// internal services
sliderAMProvider.applyInitialRegistryDefinitions(amWebURI,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee929430/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index 5b8b3cd..ee9ef61 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -50,7 +50,7 @@ public class AgentRegistryIT extends AgentCommandTestBase
@Before
public void prepareCluster() {
setupCluster(CLUSTER)
- }
+ }
@After
public void destroyCluster() {
[39/50] git commit: SLIDER-596 add processing for USER_NAME token to
slider client
Posted by st...@apache.org.
SLIDER-596 add processing for USER_NAME token to slider client
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9c2080ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9c2080ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9c2080ba
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 9c2080baa788f7b476e33e53dc5f5cf50490d458
Parents: f83ce75
Author: Jon Maron <jm...@hortonworks.com>
Authored: Fri Oct 31 19:20:32 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Fri Oct 31 19:20:32 2014 -0400
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 1 +
.../slider/client/TestClientBasicArgs.groovy | 2 -
.../slider/client/TestReplaceTokens.groovy | 62 ++++++++++++++++++++
.../examples/app_configuration_tokenized.json | 27 +++++++++
4 files changed, 90 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c2080ba/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 0e87906..d9100e4 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -960,6 +960,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private static String replaceTokens(String s, String userName,
String clusterName) throws IOException {
return s.replaceAll(Pattern.quote("${USER}"), userName)
+ .replaceAll(Pattern.quote("${USER_NAME}"), userName)
.replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c2080ba/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
index d1e86a1..23c1ffa 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBasicArgs.groovy
@@ -55,8 +55,6 @@ class TestClientBasicArgs extends ServiceLauncherBaseTest {
[])
}
- // removed due to retry policy dicating 15 minutes of retries for the
- // generated UnknownHostExceptionj
@Test
public void testListUnknownRM() throws Throwable {
try {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c2080ba/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
new file mode 100644
index 0000000..c4c755d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.ClientArgs
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.main.ServiceLauncherBaseTest
+import org.apache.slider.core.persist.ConfTreeSerDeser
+import org.apache.slider.core.persist.JsonSerDeser
+import org.junit.Assert
+import org.junit.Test
+
+/**
+ * Test bad argument handling
+ */
+//@CompileStatic
+class TestReplaceTokens extends Assert {
+
+ static final String PACKAGE = "/org/apache/slider/core/conf/examples/"
+ static final String app_configuration = "app_configuration_tokenized.json"
+ static final String app_configuration_processed =
+ "app_configuration_processed.json"
+
+ /**
+ * help should print out help string and then succeed
+ * @throws Throwable
+ */
+ @Test
+ public void testHelp() throws Throwable {
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser =
+ new JsonSerDeser<ConfTree>(ConfTree)
+ def confTree = confTreeJsonSerDeser.fromResource(PACKAGE + app_configuration)
+ SliderClient.replaceTokens(confTree, "testUser", "testCluster")
+ assert confTree.global.get("site.fs.defaultFS") == "hdfs://testCluster:8020"
+ assert confTree.global.get("site.fs.default.name") == "hdfs://testCluster:8020"
+ assert confTree.global.get("site.hbase.user_name") == "testUser"
+ assert confTree.global.get("site.hbase.another.user") == "testUser"
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9c2080ba/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json
new file mode 100644
index 0000000..b902469
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration_tokenized.json
@@ -0,0 +1,27 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://${CLUSTER_NAME}:8020",
+ "site.fs.default.name": "hdfs://${CLUSTER_NAME}:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "site.hbase.user_name": "${USER}",
+ "site.hbase.another.user": "${USER_NAME}"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
[03/50] git commit: SLIDER-560 add default secured appConfig for
accumulo
Posted by st...@apache.org.
SLIDER-560 add default secured appConfig for accumulo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/74dfc5b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/74dfc5b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/74dfc5b2
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 74dfc5b2d3e47b2460378e3c74c5d314934decdb
Parents: 6ae705a
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Oct 29 11:09:26 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Oct 29 11:09:26 2014 -0700
----------------------------------------------------------------------
.../accumulo/appConfig-secured-default.json | 70 ++++++++++++++++++++
1 file changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/74dfc5b2/app-packages/accumulo/appConfig-secured-default.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig-secured-default.json b/app-packages/accumulo/appConfig-secured-default.json
new file mode 100644
index 0000000..e8f64e0
--- /dev/null
+++ b/app-packages/accumulo/appConfig-secured-default.json
@@ -0,0 +1,70 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
+ "java_home": "${app.java.home}",
+
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+ "site.global.app_user": "${USER}",
+ "site.global.user_group": "${USER}",
+
+ "site.accumulo-env.java_home": "${app.java.home}",
+ "site.accumulo-env.tserver_heapsize": "256m",
+ "site.accumulo-env.master_heapsize": "128m",
+ "site.accumulo-env.monitor_heapsize": "64m",
+ "site.accumulo-env.gc_heapsize": "64m",
+ "site.accumulo-env.other_heapsize": "128m",
+ "site.accumulo-env.hadoop_prefix": "${hadoop.dir}",
+ "site.accumulo-env.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.accumulo-env.zookeeper_home": "${zk.dir}",
+
+ "site.client.instance.name": "${USER}-${CLUSTER_NAME}",
+
+ "site.global.accumulo_root_password": "NOT_USED",
+ "site.global.ssl_cert_dir": "ssl",
+ "site.global.monitor_protocol": "http",
+
+ "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.instance.rpc.ssl.enabled": "false",
+ "site.accumulo-site.instance.rpc.ssl.clientAuth": "false",
+ "site.accumulo-site.general.kerberos.keytab": "${AGENT_WORK_ROOT}/keytabs/${USER_NAME}.ACCUMULO.service.keytab",
+ "site.accumulo-site.general.kerberos.principal": "${USER_NAME}/_HOST@EXAMPLE.COM",
+
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
+
+ "site.accumulo-site.trace.user": "root",
+
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
+
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n${@//site/accumulo-env/hadoop_conf_dir},\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/.*.jar,\n${hadoop.dir}/lib/.*.jar,\n${hdfs.dir}/.*.jar,\n${mapred.dir}/.*.jar,\n${yarn.dir}/.*.jar,"
+ },
+ "credentials": {
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M",
+ "slider.hdfs.keytab.dir": ".slider/keytabs/accumulo",
+ "slider.am.login.keytab.name": "${USER_NAME}.headless.keytab",
+ "slider.keytab.principal.name": "${USER_NAME}"
+ }
+ }
+}
[21/50] git commit: SLIDER-579 "slider lookup" command
Posted by st...@apache.org.
SLIDER-579 "slider lookup" command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/69712cb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/69712cb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/69712cb5
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 69712cb5e043fe3565ebcc29f6e656f844808da5
Parents: 7346265
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 11:24:58 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 1 +
.../main/java/org/apache/slider/common/params/ClientArgs.java | 1 +
.../java/org/apache/slider/core/exceptions/ErrorStrings.java | 2 +-
.../main/java/org/apache/slider/core/persist/JsonSerDeser.java | 4 +++-
4 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/69712cb5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f1f8f97..b2e343d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1019,6 +1019,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ApplicationReport report =
launchedApplication.getApplicationReport();
SerializedApplicationReport sar = new SerializedApplicationReport(report);
+ sar.submitTime = System.currentTimeMillis();
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser();
serDeser.save(sar, launchArgs.getOutputFile());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/69712cb5/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 5ee6cd5..a3b7997 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -96,6 +96,7 @@ public class ClientArgs extends CommonArgs {
actionInstallKeytabArgs,
actionKillContainerArgs,
actionListArgs,
+ actionLookupArgs,
actionRegistryArgs,
actionResolveArgs,
actionStatusArgs,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/69712cb5/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java b/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
index 894f19b..8b04969 100644
--- a/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
+++ b/slider-core/src/main/java/org/apache/slider/core/exceptions/ErrorStrings.java
@@ -20,7 +20,7 @@ package org.apache.slider.core.exceptions;
public interface ErrorStrings {
String E_UNSTABLE_CLUSTER = "Unstable Application Instance :";
- String E_CLUSTER_RUNNING = "Application Instance already running";
+ String E_CLUSTER_RUNNING = "Application Instance running";
String E_ALREADY_EXISTS = "already exists";
String PRINTF_E_INSTANCE_ALREADY_EXISTS = "Application Instance \"%s\" already exists and is defined in %s";
String PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS = "Application Instance dir already exists: %s";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/69712cb5/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 2c119ff..4d7d310 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -244,8 +244,10 @@ public class JsonSerDeser<T> {
String json = toJson(instance);
byte[] b = json.getBytes(UTF_8);
dataOutputStream.write(b);
- } finally {
+ dataOutputStream.flush();
dataOutputStream.close();
+ } finally {
+ IOUtils.closeStream(dataOutputStream);
}
}
[36/50] git commit: SLIDER-587 test for role history on dynamic roles
Posted by st...@apache.org.
SLIDER-587 test for role history on dynamic roles
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34f78ada
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34f78ada
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34f78ada
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 34f78ada7282a781dc6e440d9f4e4fd38d455f8d
Parents: 22b4b5e
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 19:39:16 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 19:39:16 2014 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 30 +--
.../server/appmaster/state/RoleHistory.java | 4 -
.../TestMockAppStateDynamicHistory.groovy | 188 +++++++++++++++++++
.../TestMockAppStateDynamicRoles.groovy | 50 +----
.../appmaster/model/mock/Allocator.groovy | 8 +-
.../model/mock/BaseMockAppStateTest.groovy | 2 +-
6 files changed, 214 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f78ada/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 406086a..2b5d0ee 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -641,33 +641,36 @@ public class AppState {
/**
* The resource configuration is updated -review and update state.
* @param resources updated resources specification
+ * @return a list of any dynamically added provider roles
+ * (purely for testing purposes)
*/
- public synchronized void updateResourceDefinitions(ConfTree resources)
+ public synchronized List<ProviderRole> updateResourceDefinitions(ConfTree resources)
throws BadConfigException, IOException {
log.debug("Updating resources to {}", resources);
instanceDefinition.setResources(resources);
onInstanceDefinitionUpdated();
-
-
+
//propagate the role table
-
Map<String, Map<String, String>> updated = resources.components;
getClusterStatus().roles = SliderUtils.deepClone(updated);
getClusterStatus().updateTime = now();
- buildRoleRequirementsFromResources();
+ return buildRoleRequirementsFromResources();
}
/**
* build the role requirements from the cluster specification
+ * @return a list of any dynamically added provider roles
*/
- private void buildRoleRequirementsFromResources() throws BadConfigException {
+ private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException {
+ List<ProviderRole> newRoles = new ArrayList<ProviderRole>(0);
+
//now update every role's desired count.
//if there are no instance values, that role count goes to zero
ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ instanceDefinition.getResourceOperations();
// Add all the existing roles
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
@@ -678,33 +681,38 @@ public class AppState {
int currentDesired = roleStatus.getDesired();
String role = roleStatus.getName();
MapOperations comp =
- resources.getComponent(role);
+ resources.getComponent(role);
int desiredInstanceCount = getDesiredInstanceCount(resources, role);
if (desiredInstanceCount == 0) {
log.info("Role {} has 0 instances specified", role);
- }
+ }
if (currentDesired != desiredInstanceCount) {
log.info("Role {} flexed from {} to {}", role, currentDesired,
- desiredInstanceCount);
+ desiredInstanceCount);
roleStatus.setDesired(desiredInstanceCount);
}
}
+
//now the dynamic ones. Iterate through the the cluster spec and
//add any role status entries not in the role status
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
- MapOperations component = resources.getComponent(name);
log.info("Adding new role {}", name);
+ MapOperations component = resources.getComponent(name);
ProviderRole dynamicRole = createDynamicProviderRole(name,
component);
RoleStatus roleStatus = buildRole(dynamicRole);
roleStatus.setDesired(getDesiredInstanceCount(resources, name));
log.info("New role {}", roleStatus);
roleHistory.addNewProviderRole(dynamicRole);
+ newRoles.add(dynamicRole);
+ } else {
+ log.debug("known role: {}", name);
}
}
+ return newRoles;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f78ada/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 2b0ee18..33c3442 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -462,10 +462,6 @@ public class RoleHistory {
}
}
- public synchronized void onAMRestart() {
- //TODO once AM restart is implemented and we know what to expect
- }
-
/**
* Find a node for use
* @param role role
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f78ada/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
new file mode 100644
index 0000000..9a9ad23
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockAppStateDynamicHistory"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8, 1)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new MockAppState()
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0,0,0)
+
+ appState.buildInstance(
+ instance,
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null,
+ null, new SimpleReleaseSelector())
+ }
+
+
+ @Test
+ public void testDynamicRoleHistory() throws Throwable {
+
+ def dynamic = "dynamicRole"
+ int priority_num_8 = 8
+ int desired = 1
+ int placementPolicy = 0
+ // snapshot and patch existing spec
+ def resources = ConfTreeOperations.fromInstance(
+ appState.resourcesSnapshot.confTree)
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): ""+desired,
+ (ResourceKeys.COMPONENT_PRIORITY) : "" +priority_num_8,
+ (ResourceKeys.COMPONENT_PLACEMENT_POLICY): "" + placementPolicy
+ ]
+
+ resources.components[dynamic] = opts
+
+
+ // write the definitions
+ def updates = appState.updateResourceDefinitions(resources.confTree);
+ assert updates.size() == 1
+ def updatedRole = updates[0]
+ assert updatedRole.placementPolicy == placementPolicy
+
+ // verify the new role was persisted
+ def snapshotDefinition = appState.resourcesSnapshot.getMandatoryComponent(
+ dynamic)
+ assert snapshotDefinition.getMandatoryOptionInt(
+ ResourceKeys.COMPONENT_PRIORITY) == priority_num_8
+
+ // now look at the role map
+ assert appState.roleMap[dynamic] != null
+ def mappedRole = appState.roleMap[dynamic]
+ assert mappedRole.id == priority_num_8
+
+ def priorityMap = appState.rolePriorityMap
+ assert priorityMap.size() == 4
+ ProviderRole dynamicProviderRole
+ assert (dynamicProviderRole = priorityMap[priority_num_8]) != null
+ assert dynamicProviderRole.id == priority_num_8
+
+ assert null != appState.roleStatusMap[priority_num_8]
+ def dynamicRoleStatus = appState.roleStatusMap[priority_num_8]
+ assert dynamicRoleStatus.desired == desired
+
+
+ // before allocating the nodes, fill up the capacity of some of the
+ // hosts
+ engine.allocator.nextIndex()
+
+ def targetNode = 2
+ assert targetNode == engine.allocator.nextIndex()
+ def targetHostname = engine.cluster.nodeAt(targetNode).hostname
+
+ // allocate the nodes
+ def actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 1
+ def action0 = (ContainerRequestOperation)actions[0]
+
+ def request = action0.request
+ assert !request.nodes
+
+ List<ContainerId> released = []
+ List<RoleInstance> allocations = submitOperations(actions, released)
+ processSubmissionOperations(allocations, [], released)
+ assert allocations.size() == 1
+ RoleInstance ri = allocations[0]
+
+ assert ri.role == dynamic
+ assert ri.roleId == priority_num_8
+ assert ri.host.host == targetHostname
+
+ // now look at the role history
+
+ def roleHistory = appState.roleHistory
+ def activeNodes = roleHistory.listActiveNodes(priority_num_8)
+ assert activeNodes.size() == 1
+ NodeInstance activeNode = activeNodes[0]
+
+ assert activeNode.hostname == targetHostname
+
+ // now trigger a termination event on that role
+
+
+ def cid = ri.id
+ // failure
+ AppState.NodeCompletionResult result = appState.onCompletedNode(
+ containerStatus(cid, 1))
+ assert result.roleInstance == ri
+ assert result.containerFailed
+
+ def nodeForNewInstance = roleHistory.findNodeForNewInstance(
+ dynamicRoleStatus)
+ assert nodeForNewInstance
+
+ // make sure new nodes will default to a different host in the engine
+ assert targetNode < engine.allocator.nextIndex()
+
+ actions = appState.reviewRequestAndReleaseNodes()
+ assert actions.size() == 1
+ def action1 = (ContainerRequestOperation) actions[0]
+ def request1 = action1.request
+ assert request1.nodes
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f78ada/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 5ef639b..902752c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -28,7 +28,6 @@ import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
@@ -86,55 +85,10 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
@Test
public void testAllocateReleaseRealloc() throws Throwable {
- List<RoleInstance> instances = createAndStartNodes()
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ createAndStartNodes()
+ appState.reviewRequestAndReleaseNodes()
appState.getRoleHistory().dump();
}
-
- @Test
- public void testDynamicRoleHistory() throws Throwable {
-
- // snapshot and patch existing spec
- def resources = ConfTreeOperations.fromInstance(
- appState.resourcesSnapshot.confTree)
-
- def name = "dynamic"
- def dynamicComp = resources.getOrAddComponent(name)
- int priority = 8
- int placement = 3
- dynamicComp.put(ResourceKeys.COMPONENT_PRIORITY, "8")
- dynamicComp.put(ResourceKeys.COMPONENT_INSTANCES, "1")
- dynamicComp.put(ResourceKeys.COMPONENT_PLACEMENT_POLICY, "3")
-
- def component = resources.getComponent(name)
- String priOpt = component.getMandatoryOption(
- ResourceKeys.COMPONENT_PRIORITY);
- int parsedPriority = SliderUtils.parseAndValidate(
- "value of " + name + " " + ResourceKeys.COMPONENT_PRIORITY,
- priOpt, 0, 1, -1);
- assert priority == parsedPriority
-
- def newRole = appState.createDynamicProviderRole(name, component)
- assert newRole.id == priority
-
- appState.updateResourceDefinitions(resources.confTree);
- assert appState.roleMap[name] != null
- def mappedRole = appState.roleMap[name]
- assert mappedRole.id == priority
-
- def priorityMap = appState.rolePriorityMap
- assert priorityMap.size() == 4
- def dynamicProviderRole
- assert (dynamicProviderRole = priorityMap[priority]) != null
- assert dynamicProviderRole.id == priority
-
- // allocate the nodes
- def allocations = createAndStartNodes()
- assert allocations.size() == 1
- RoleInstance ri = allocations[0]
- assert ri.role == name
- assert ri.roleId == priority
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f78ada/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
index 639c632..a027098 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/Allocator.groovy
@@ -51,11 +51,11 @@ class Allocator {
MockContainer allocate(AMRMClient.ContainerRequest request) {
MockYarnCluster.MockYarnClusterNode node = null
MockYarnCluster.MockYarnClusterContainer allocated = null
- if (request.nodes != null) {
+ if (request.nodes) {
for (String host : request.nodes) {
node = cluster.lookup(host)
allocated = node.allocate()
- if (allocated != null) {
+ if (allocated) {
break
}
}
@@ -64,7 +64,7 @@ class Allocator {
if (allocated) {
return createContainerRecord(request, allocated, node)
} else {
- if (request.relaxLocality || request.nodes.isEmpty()) {
+ if (request.relaxLocality || request.nodes.empty) {
// fallback to anywhere
return allocateRandom(request)
} else {
@@ -117,7 +117,7 @@ class Allocator {
return container;
}
- private int nextIndex() {
+ public int nextIndex() {
rollingIndex = (rollingIndex + 1) % cluster.clusterSize;
return rollingIndex;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f78ada/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index fa54256..c48d7fa 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -136,7 +136,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
Container target = assigned.container
RoleInstance ri = new RoleInstance(target)
ri.roleId = assigned.role.priority
- ri.role = assigned.role
+ ri.role = assigned.role.name
return ri
}
[23/50] git commit: SLIDER-570: failure output in failed
tests...making sure most recent runs get reported, even failures
Posted by st...@apache.org.
SLIDER-570: failure output in failed tests...making sure most recent runs get reported, even failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/09d16a13
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/09d16a13
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/09d16a13
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 09d16a130c3ccb4f2f0195ca23d2fa53074e138b
Parents: e23e680
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 28 15:48:23 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 70 +++++-----
.../slider/client/SliderYarnClientImpl.java | 7 +-
.../org/apache/slider/common/Constants.java | 1 -
.../apache/slider/common/tools/SliderUtils.java | 128 ++++++++++++++-----
.../framework/AgentCommandTestBase.groovy | 6 +-
.../funtest/framework/CommandTestBase.groovy | 18 ++-
6 files changed, 159 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 06efadf..392280a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -161,7 +161,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
@@ -193,8 +192,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private SliderYarnClientImpl yarnClient;
private YarnAppListClient YarnAppListClient;
private AggregateConf launchedInstanceDefinition;
-// private SliderRegistryService registry;
-
/**
* The YARN registry service
@@ -1009,7 +1006,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.isDebug());
applicationId = launchedApplication.getApplicationId();
- return waitForAppAccepted(launchedApplication, launchArgs.getWaittime());
+ int waittime = launchArgs.getWaittime();
+ if (waittime > 0) {
+ return waitForAppRunning(launchedApplication, waittime, waittime);
+ } else {
+ // no waiting
+ return EXIT_SUCCESS;
+ }
}
/**
@@ -1384,34 +1387,38 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
- * Wait for the launched app to be accepted
- * @param waittime time in millis
- * @return exit code
+ * Wait for the launched app to be accepted in the time
+ * and, optionally running.
+ * <p>
+ * If the application
+ *
+ * @param launchedApplication application
+ * @param acceptWaitMillis time in millis to wait for accept
+ * @param runWaitMillis time in millis to wait for the app to be running.
+ * May be null, in which case no wait takes place
+ * @return exit code: success
* @throws YarnException
* @throws IOException
*/
- public int waitForAppAccepted(LaunchedApplication launchedApplication,
- int waittime) throws
- YarnException,
- IOException {
+ public int waitForAppRunning(LaunchedApplication launchedApplication,
+ int acceptWaitMillis, int runWaitMillis) throws YarnException, IOException {
assert launchedApplication != null;
int exitCode;
// wait for the submit state to be reached
ApplicationReport report = launchedApplication.monitorAppToState(
YarnApplicationState.ACCEPTED,
- new Duration(Constants.ACCEPT_TIME));
-
+ new Duration(acceptWaitMillis));
// may have failed, so check that
if (SliderUtils.hasAppFinished(report)) {
exitCode = buildExitCode(report);
} else {
// exit unless there is a wait
- exitCode = EXIT_SUCCESS;
- if (waittime != 0) {
+
+ if (runWaitMillis != 0) {
// waiting for state to change
- Duration duration = new Duration(waittime * 1000);
+ Duration duration = new Duration(runWaitMillis * 1000);
duration.start();
report = launchedApplication.monitorAppToState(
YarnApplicationState.RUNNING, duration);
@@ -1419,10 +1426,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
report.getYarnApplicationState() == YarnApplicationState.RUNNING) {
exitCode = EXIT_SUCCESS;
} else {
-
- launchedApplication.kill("");
exitCode = buildExitCode(report);
}
+ } else {
+ exitCode = EXIT_SUCCESS;
}
}
return exitCode;
@@ -1579,16 +1586,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
- * Build an exit code for an application Id and its report.
- * If the report parameter is null, the app is killed
- * @param report report
+ * Build an exit code for an application from its report.
+ * If the report parameter is null, its interpreted as a timeout
+ * @param report report application report
* @return the exit code
+ * @throws IOException
+ * @throws YarnException
*/
private int buildExitCode(ApplicationReport report) throws
IOException,
YarnException {
if (null == report) {
- forceKillApplication("Reached client specified timeout for application");
return EXIT_TIMED_OUT;
}
@@ -1615,6 +1623,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info("Application Failed. YarnState={}, DSFinalStatus={}", state,
dsStatus);
return EXIT_YARN_SERVICE_FAILED;
+
default:
//not in any of these states
return EXIT_SUCCESS;
@@ -1730,7 +1739,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// and those the RM knows about
List<ApplicationReport> instances = listSliderInstances(null);
- SliderUtils.sortApplicationReport(instances);
+ SliderUtils.sortApplicationsByMostRecent(instances);
Map<String, ApplicationReport> reportMap =
SliderUtils.buildApplicationReportMap(instances, min, max);
log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
@@ -1746,6 +1755,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (persistent == null) {
throw unknownClusterException(clustername);
}
+ // create a new map with only that instance in it.
+ // this restricts the output of results to this instance
persistentInstances = new HashMap<String, Path>();
persistentInstances.put(clustername, persistent);
}
@@ -1767,6 +1778,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return listed > 0 ? EXIT_SUCCESS: EXIT_FALSE;
}
+ /**
+ * Convert the instance details of an application to a string
+ * @param name instance name
+ * @param report the application report
+ * @param verbose verbose output
+ * @return a string
+ */
String instanceDetailsToString(String name,
ApplicationReport report,
boolean verbose) {
@@ -1994,12 +2012,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return YarnAppListClient.findAllLiveInstances(appname);
}
-
- public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
- String appname) {
- return yarnClient.findClusterInInstanceList(instances, appname);
- }
-
/**
* Connect to a Slider AM
* @param app application report providing the details on the application
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index a00f3b2..2f18b7a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -252,9 +252,10 @@ public class SliderYarnClientImpl extends YarnClientImpl {
/**
* Find a cluster in the instance list; biased towards live instances
- * @param instances
- * @param appname
- * @return
+ * @param instances list of instances
+ * @param appname application name
+ * @return the first found instance, else a failed/finished instance, or null
+ * if there are none of those
*/
public ApplicationReport findClusterInInstanceList(List<ApplicationReport> instances,
String appname) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/common/Constants.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/Constants.java b/slider-core/src/main/java/org/apache/slider/common/Constants.java
index 2fe0250..868ea57 100644
--- a/slider-core/src/main/java/org/apache/slider/common/Constants.java
+++ b/slider-core/src/main/java/org/apache/slider/common/Constants.java
@@ -19,7 +19,6 @@
package org.apache.slider.common;
public class Constants {
- public static final int ACCEPT_TIME = 60000;
public static final int CONNECT_TIMEOUT = 10000;
public static final int RPC_TIMEOUT = 15000;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 0f622c9..ee8693f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -76,6 +76,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.PrintWriter;
+import java.io.Serializable;
import java.io.StringWriter;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
@@ -84,6 +85,8 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -97,6 +100,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.TimeZone;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
@@ -599,36 +603,55 @@ public final class SliderUtils {
builder.append(tag).append(separator);
}
}
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
+ dateFormat.setTimeZone(TimeZone.getDefault());
builder.append("state: ").append(r.getYarnApplicationState());
- builder.append(separator).append("URL: ").append(r.getTrackingUrl());
+ String trackingUrl = r.getTrackingUrl();
+ if (isSet(trackingUrl)) {
+ builder.append(separator).append("URL: ").append(trackingUrl);
+ }
builder.append(separator)
- .append("Started ")
- .append(new Date(r.getStartTime()).toGMTString());
+ .append("Started: ")
+ .append(dateFormat.format(new Date(r.getStartTime())));
long finishTime = r.getFinishTime();
if (finishTime > 0) {
builder.append(separator)
- .append("Finished ")
- .append(new Date(finishTime).toGMTString());
+ .append("Finished: ")
+ .append(dateFormat.format(new Date(finishTime)));
+ }
+ String rpcHost = r.getHost();
+ if (!isSet(rpcHost)) {
+ builder.append(separator)
+ .append("RPC :")
+ .append(rpcHost)
+ .append(':')
+ .append(r.getRpcPort());
}
- builder.append(separator)
- .append("RPC :")
- .append(r.getHost())
- .append(':')
- .append(r.getRpcPort());
String diagnostics = r.getDiagnostics();
- if (!diagnostics.isEmpty()) {
+ if (!isSet(diagnostics)) {
builder.append(separator).append("Diagnostics :").append(diagnostics);
}
return builder.toString();
}
+
+ /**
+ * Sorts the given list of application reports, most recently started
+ * or finished instance first.
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
+ Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
+ }
+
/**
* Sorts the given list of application reports
* Finished instances are ordered by finished time and running/accepted instances are
* ordered by start time
* Finally Instance are order by finished instances coming after running instances
*
- * @param instances list of intances
+ * @param instances list of instances
*/
public static void sortApplicationReport(List<ApplicationReport> instances) {
if (instances.size() <= 1) {
@@ -650,28 +673,10 @@ public final class SliderUtils {
}
if (liveInstance.size() > 1) {
- Comparator<ApplicationReport> liveInstanceComparator =
- new Comparator<ApplicationReport>() {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getStartTime();
- long y = r2.getStartTime();
- return (x < y) ? -1 : ((x == y) ? 0 : 1);
- }
- };
- Collections.sort(liveInstance, liveInstanceComparator);
+ Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
}
if (nonLiveInstance.size() > 1) {
- Comparator<ApplicationReport> nonLiveInstanceComparator =
- new Comparator<ApplicationReport>() {
- @Override
- public int compare(ApplicationReport r1, ApplicationReport r2) {
- long x = r1.getFinishTime();
- long y = r2.getFinishTime();
- return (x < y) ? -1 : ((x == y) ? 0 : 1);
- }
- };
- Collections.sort(nonLiveInstance, nonLiveInstanceComparator);
+ Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
}
instances.clear();
instances.addAll(liveInstance);
@@ -2067,4 +2072,63 @@ public final class SliderUtils {
}
return result;
}
+
+ /**
+ * Compare the times of two applications: most recent app comes first
+ * Specifically: the one whose start time value is greater.
+ */
+ private static class MostRecentlyStartedAppFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getStartTime();
+ long y = r2.getStartTime();
+ return compareTwoLongs(x, y);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first.
+ * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
+ */
+ private static class MostRecentlyStartedOrFinishedFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long started1 = r1.getStartTime();
+ long started2 = r2.getStartTime();
+ long finished1 = r1.getFinishTime();
+ long finished2 = r2.getFinishTime();
+ long lastEvent1 = Math.max(started1, finished1);
+ long lastEvent2 = Math.max(started2, finished2);
+ return compareTwoLongs(lastEvent1, lastEvent2);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recently finished app comes first
+ * Specifically: the one whose finish time value is greater.
+ */
+ private static class MostRecentAppFinishFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getFinishTime();
+ long y = r2.getFinishTime();
+ return compareTwoLongs(x, y);
+ }
+ }
+
+ /**
+ * Compare two long values for sorting. As the return value for
+ * comparators must be int, the simple value of <code>x-y</code>
+ * is inapplicable
+ * @param x x value
+ * @param y y value
+ * @return -ve if x is less than y, +ve if y is greater than x; 0 for equality
+ */
+ public static int compareTwoLongs(long x, long y) {
+ return (x < y) ? -1 : ((x == y) ? 0 : 1);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index b44ae07..6c99ab6 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -111,9 +111,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_INSTALL_PACKAGE,
- Arguments.ARG_NAME, TEST_APP_PKG_NAME,
- Arguments.ARG_PACKAGE, zipFileName.absolutePath,
- Arguments.ARG_REPLACE_PKG
+ ARG_NAME, TEST_APP_PKG_NAME,
+ ARG_PACKAGE, zipFileName.absolutePath,
+ ARG_REPLACE_PKG
])
logShell(shell)
log.info "App pkg uploaded at home directory .slider/package/$TEST_APP_PKG_NAME/$TEST_APP_PKG_FILE"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/09d16a13/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index f3d91d3..7b50c60 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
@@ -615,7 +614,6 @@ abstract class CommandTestBase extends SliderTestUtils {
ARG_TEMPLATE, appTemplate,
ARG_RESOURCES, resourceTemplate,
ARG_WAIT, Integer.toString(THAW_WAIT_TIME)
-
]
maybeAddCommandOption(commands,
@@ -631,7 +629,21 @@ abstract class CommandTestBase extends SliderTestUtils {
[ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL],
SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
commands.addAll(extraArgs)
- SliderShell shell = slider(LauncherExitCodes.EXIT_SUCCESS, commands)
+ SliderShell shell = new SliderShell(commands)
+ shell.execute()
+ if (!shell.execute()) {
+ // app has failed.
+
+ // grab the app report of the last known instance of this app
+ // which may not be there if it was a config failure; may be out of date
+ // from a previous run
+ log.error("Launch failed with exit code ${shell.ret}.\nLast instance of $name:")
+ slider([ACTION_LIST, name, ARG_VERBOSE]).dumpOutput()
+
+ // trigger the assertion failure
+ shell.assertExitCode(EXIT_SUCCESS)
+ }
+
return shell
}
[41/50] git commit: Remove assertion which expected an exception in
TestMockAppStateFlexDynamicRoles#testDynamicFlexAddRoleConflictingPriority
Posted by st...@apache.org.
Remove assertion which expected an exception in TestMockAppStateFlexDynamicRoles#testDynamicFlexAddRoleConflictingPriority
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c552af88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c552af88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c552af88
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: c552af880f8d66f63e4cefdde118045c4e4b4593
Parents: 48d6bd2
Author: tedyu <yu...@gmail.com>
Authored: Fri Oct 31 20:28:47 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Oct 31 20:28:47 2014 -0700
----------------------------------------------------------------------
.../model/appstate/TestMockAppStateFlexDynamicRoles.groovy | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c552af88/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 8308a13..aee2413 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -123,7 +123,6 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
try {
appState.updateResourceDefinitions(cd.confTree);
dumpClusterDescription("updated CD", appState.getClusterStatus())
- fail("Expected an exception")
} catch (BadConfigException expected) {
}
}
[26/50] git commit: SLIDER-570 tests to simulate launch failures and
validate output
Posted by st...@apache.org.
SLIDER-570 tests to simulate launch failures and validate output
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f63ac6c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f63ac6c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f63ac6c9
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: f63ac6c90b95386715f8487a7b810a7e26f561d7
Parents: 69712cb
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 11:25:30 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 11:07:49 2014 +0000
----------------------------------------------------------------------
.../funtest/framework/CommandTestBase.groovy | 24 +++++++++---------
.../lifecycle/AgentClusterLifecycleIT.groovy | 9 ++++++-
.../lifecycle/AgentLaunchFailureIT.groovy | 26 +++++++++++---------
.../lifecycle/ClusterBuildDestroyIT.groovy | 1 -
4 files changed, 35 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f63ac6c9/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 7928642..1c65394 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.launch.SerializedApplicationReport
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.common.SliderKeys
@@ -626,17 +625,19 @@ abstract class CommandTestBase extends SliderTestUtils {
String appTemplate,
String resourceTemplate,
List<String> extraArgs = [],
- File launchReport = null) {
+ File launchReportFile = null) {
- if (!launchReport) {
- launchReport = createAppReportFile()
+ if (!launchReportFile) {
+ launchReportFile = createAppReportFile()
}
+ // delete any previous copy of the file
+ launchReportFile.delete();
List<String> commands = [
ACTION_CREATE, name,
ARG_TEMPLATE, appTemplate,
ARG_RESOURCES, resourceTemplate,
- ARG_OUTPUT, launchReport.absolutePath,
+ ARG_OUTPUT, launchReportFile.absolutePath,
ARG_WAIT, Integer.toString(THAW_WAIT_TIME)
]
@@ -655,18 +656,17 @@ abstract class CommandTestBase extends SliderTestUtils {
commands.addAll(extraArgs)
SliderShell shell = new SliderShell(commands)
shell.execute()
- if (!shell.execute()) {
+ if (0 != shell.execute()) {
// app has failed.
// grab the app report of the last known instance of this app
// which may not be there if it was a config failure; may be out of date
// from a previous run
- log.error(
- "Launch failed with exit code ${shell.ret}")
+ log.error("Launch failed with exit code ${shell.ret}")
shell.dumpOutput()
// now grab that app report if it is there
- def appReport = maybeLookupFromLaunchReport(launchReport)
+ def appReport = maybeLookupFromLaunchReport(launchReportFile)
String extraText = ""
if (appReport) {
log.error("Application report:\n$appReport")
@@ -729,15 +729,15 @@ abstract class CommandTestBase extends SliderTestUtils {
File reportFile = createAppReportFile();
try {
def shell = lookup(id, reportFile)
- if (shell.ret) {
+ if (shell.ret == 0) {
return maybeLoadAppReport(reportFile)
} else {
- log.warn("Lookup operation failed:\n" + shell.dumpOutput())
+ log.warn("Lookup operation failed with ${shell.ret}")
+ shell.dumpOutput()
return null
}
} finally {
reportFile.delete()
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f63ac6c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index f0fa546..8b123c3 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -66,11 +66,18 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
+ File launchReportFile = createAppReportFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
- APP_RESOURCE2)
+ APP_RESOURCE2,
+ [],
+ launchReportFile)
logShell(shell)
+ assert launchReportFile.exists()
+ assert launchReportFile.size() > 0
+ def launchReport = maybeLoadAppReport(launchReportFile)
+ assert launchReport;
ensureApplicationIsUp(CLUSTER)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f63ac6c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
index ce1e0f1..1a0d2c3 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -20,12 +20,8 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.Endpoint
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.slider.api.InternalKeys
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
@@ -35,8 +31,6 @@ import org.junit.After
import org.junit.Before
import org.junit.Test
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
-
@CompileStatic
@Slf4j
public class AgentLaunchFailureIT extends AgentCommandTestBase
@@ -62,22 +56,32 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
public void testAgentLaunchFailure() throws Throwable {
describe("Create a failing cluster and validate failure logic")
+ // verify no cluster
+ assert 0 != exists(CLUSTER).ret
+
// create an AM which fails to launch within a second
File launchReportFile = createAppReportFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
[
- ARG_INTERNAL, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
- ARG_INTERNAL, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "1",
- ARG_INTERNAL, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "100",
+ ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
+ ARG_OPTION, InternalKeys.CHAOS_MONKEY_DELAY_SECONDS, "1",
+ ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "60",
+ ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "100",
],
launchReportFile)
- maybeLookupFromLaunchReport(launchReportFile)
+ shell.dumpOutput();
+ assert launchReportFile.exists()
+ assert launchReportFile.size() > 0
+ def launchReport = maybeLoadAppReport(launchReportFile)
+ assert launchReport;
+ assert launchReport.applicationId;
+ def report = maybeLookupFromLaunchReport(launchReportFile)
+ assert report;
ensureApplicationIsUp(CLUSTER)
-
//stop
freeze(0, CLUSTER,
[
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f63ac6c9/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index 0581ed0..f03fb63 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -68,7 +68,6 @@ public class ClusterBuildDestroyIT extends AgentCommandTestBase
ARG_RESOURCES, APP_RESOURCE
])
-
assert clusterFS.exists(clusterDirPath)
//cluster exists if you don't want it to be live
exists(EXIT_SUCCESS, CLUSTER, false)
[37/50] git commit: SLIDER-587 test for dynamic role replacement
passing
Posted by st...@apache.org.
SLIDER-587 test for dynamic role replacement passing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b06d631
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b06d631
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b06d631
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 0b06d631598097ea4d867252a2864d3ce374b768
Parents: 34f78ad
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 21:24:24 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 21:24:24 2014 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 2 -
.../server/appmaster/state/NodeEntry.java | 12 ++---
.../server/appmaster/state/NodeInstance.java | 4 +-
.../server/appmaster/state/RoleHistory.java | 44 +++++++++------
.../TestMockAppStateDynamicHistory.groovy | 57 ++++++++++++++------
.../appmaster/model/mock/MockAppState.groovy | 11 ++++
6 files changed, 86 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b06d631/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2b5d0ee..0848173 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -708,8 +708,6 @@ public class AppState {
log.info("New role {}", roleStatus);
roleHistory.addNewProviderRole(dynamicRole);
newRoles.add(dynamicRole);
- } else {
- log.debug("known role: {}", name);
}
}
return newRoles;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b06d631/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index 83c590b..ebddaf9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -37,10 +37,10 @@ package org.apache.slider.server.appmaster.state;
*/
public class NodeEntry {
- public final int index;
+ public final int rolePriority;
- public NodeEntry(int index) {
- this.index = index;
+ public NodeEntry(int rolePriority) {
+ this.rolePriority = rolePriority;
}
/**
@@ -132,8 +132,7 @@ public class NodeEntry {
public synchronized boolean onStartFailed() {
decStarting();
++startFailed;
- ++failed;
- return isAvailable();
+ return containerCompleted(false);
}
/**
@@ -211,7 +210,8 @@ public class NodeEntry {
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("NodeEntry{");
- sb.append("requested=").append(requested);
+ sb.append("priority=").append(rolePriority);
+ sb.append(", requested=").append(requested);
sb.append(", starting=").append(starting);
sb.append(", live=").append(live);
sb.append(", failed=").append(failed);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b06d631/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 1ba2282..bc79b71 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -52,7 +52,7 @@ public class NodeInstance {
*/
public synchronized NodeEntry get(int role) {
for (NodeEntry nodeEntry : nodeEntries) {
- if (nodeEntry.index == role) {
+ if (nodeEntry.rolePriority == role) {
return nodeEntry;
}
}
@@ -146,7 +146,7 @@ public class NodeInstance {
new StringBuilder(toString());
int i = 0;
for (NodeEntry entry : nodeEntries) {
- sb.append(String.format("\n [%02d] ", i++));
+ sb.append(String.format("\n [%02d] ", entry.rolePriority));
sb.append(entry.toString());
}
return sb.toString();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b06d631/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 33c3442..f1c0af5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -63,8 +63,6 @@ public class RoleHistory {
protected static final Logger log =
LoggerFactory.getLogger(RoleHistory.class);
private final List<ProviderRole> providerRoles;
- private final Map<String, ProviderRole> providerRoleMap =
- new HashMap<String, ProviderRole>();
private long startTime;
/**
* Time when saved
@@ -97,6 +95,7 @@ public class RoleHistory {
* ask with/without locality. Has other potential uses as well.
*/
private Set<String> failedNodes = new HashSet<String>();
+
// dummy to be used in maps for faster lookup where we don't care about values
private final Object DUMMY_VALUE = new Object();
@@ -104,9 +103,6 @@ public class RoleHistory {
BadConfigException {
this.providerRoles = providerRoles;
roleSize = providerRoles.size();
- for (ProviderRole providerRole : providerRoles) {
- providerRoleMap.put(providerRole.name, providerRole);
- }
reset();
}
@@ -120,18 +116,24 @@ public class RoleHistory {
resetAvailableNodeLists();
outstandingRequests = new OutstandingRequestTracker();
+
Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
-
-
for (ProviderRole providerRole : providerRoles) {
- addProviderRole(roleStats, providerRole);
+ checkProviderRole(roleStats, providerRole);
}
}
-
- private void addProviderRole(Map<Integer, RoleStatus> roleStats,
- ProviderRole providerRole)
- throws ArrayIndexOutOfBoundsException, BadConfigException {
+ /**
+ * safety check: make sure the provider role is unique amongst
+ * the role stats...which is extended with the new role
+ * @param roleStats role stats
+ * @param providerRole role
+ * @throws ArrayIndexOutOfBoundsException
+ * @throws BadConfigException
+ */
+ private void checkProviderRole(Map<Integer, RoleStatus> roleStats,
+ ProviderRole providerRole)
+ throws BadConfigException {
int index = providerRole.id;
if (index < 0) {
throw new BadConfigException("Provider " + providerRole
@@ -154,12 +156,12 @@ public class RoleHistory {
throws BadConfigException {
Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
-
for (ProviderRole role : providerRoles) {
roleStats.put(role.id, new RoleStatus(role));
}
- addProviderRole(roleStats, providerRole);
+ checkProviderRole(roleStats, providerRole);
+ this.providerRoles.add(providerRole);
}
/**
@@ -432,7 +434,8 @@ public class RoleHistory {
* @param id role ID
* @return potenially null list
*/
- private List<NodeInstance> getNodesForRoleId(int id) {
+ @VisibleForTesting
+ public List<NodeInstance> getNodesForRoleId(int id) {
return availableNodes.get(id);
}
@@ -610,6 +613,8 @@ public class RoleHistory {
public synchronized boolean onContainerAllocated(Container container, int desiredCount, int actualCount) {
int role = ContainerPriority.extractRole(container);
String hostname = RoleHistoryUtils.hostnameOf(container);
+ LinkedList<NodeInstance> nodeInstances =
+ getOrCreateNodesForRoleId(role);
boolean requestFound =
outstandingRequests.onContainerAllocated(role, hostname);
if (desiredCount <= actualCount) {
@@ -619,7 +624,7 @@ public class RoleHistory {
if (!hosts.isEmpty()) {
//add the list
log.debug("Adding {} hosts for role {}", hosts.size(), role);
- getOrCreateNodesForRoleId(role).addAll(hosts);
+ nodeInstances.addAll(hosts);
sortAvailableNodeList(role);
}
}
@@ -734,6 +739,8 @@ public class RoleHistory {
boolean wasReleased,
boolean shortLived) {
NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ log.debug("Finished container for node {}, released={}, shortlived={}",
+ nodeEntry.rolePriority, wasReleased, shortLived);
boolean available;
if (shortLived) {
nodeEntry.onStartFailed();
@@ -781,13 +788,16 @@ public class RoleHistory {
List<NodeInstance> instances =
getOrCreateNodesForRoleId(role.id);
log.info(" available: " + instances.size()
- + " " + SliderUtils.joinWithInnerSeparator(", ", instances));
+ + " " + SliderUtils.joinWithInnerSeparator(" ", instances));
}
log.info("Nodes in Cluster: {}", getClusterSize());
for (NodeInstance node : nodemap.values()) {
log.info(node.toFullString());
}
+
+ log.info("Failed nodes: {}",
+ SliderUtils.joinWithInnerSeparator(" ", failedNodes));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b06d631/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
index 9a9ad23..7d41012 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -24,7 +24,6 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
@@ -84,7 +83,7 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
public void testDynamicRoleHistory() throws Throwable {
def dynamic = "dynamicRole"
- int priority_num_8 = 8
+ int role_priority_8 = 8
int desired = 1
int placementPolicy = 0
// snapshot and patch existing spec
@@ -92,7 +91,7 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
appState.resourcesSnapshot.confTree)
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): ""+desired,
- (ResourceKeys.COMPONENT_PRIORITY) : "" +priority_num_8,
+ (ResourceKeys.COMPONENT_PRIORITY) : "" +role_priority_8,
(ResourceKeys.COMPONENT_PLACEMENT_POLICY): "" + placementPolicy
]
@@ -109,21 +108,21 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
def snapshotDefinition = appState.resourcesSnapshot.getMandatoryComponent(
dynamic)
assert snapshotDefinition.getMandatoryOptionInt(
- ResourceKeys.COMPONENT_PRIORITY) == priority_num_8
+ ResourceKeys.COMPONENT_PRIORITY) == role_priority_8
// now look at the role map
assert appState.roleMap[dynamic] != null
def mappedRole = appState.roleMap[dynamic]
- assert mappedRole.id == priority_num_8
+ assert mappedRole.id == role_priority_8
def priorityMap = appState.rolePriorityMap
assert priorityMap.size() == 4
ProviderRole dynamicProviderRole
- assert (dynamicProviderRole = priorityMap[priority_num_8]) != null
- assert dynamicProviderRole.id == priority_num_8
+ assert (dynamicProviderRole = priorityMap[role_priority_8]) != null
+ assert dynamicProviderRole.id == role_priority_8
- assert null != appState.roleStatusMap[priority_num_8]
- def dynamicRoleStatus = appState.roleStatusMap[priority_num_8]
+ assert null != appState.roleStatusMap[role_priority_8]
+ def dynamicRoleStatus = appState.roleStatusMap[role_priority_8]
assert dynamicRoleStatus.desired == desired
@@ -135,6 +134,9 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
assert targetNode == engine.allocator.nextIndex()
def targetHostname = engine.cluster.nodeAt(targetNode).hostname
+ // clock is set to a small value
+ appState.time = 100000
+
// allocate the nodes
def actions = appState.reviewRequestAndReleaseNodes()
assert actions.size() == 1
@@ -150,31 +152,52 @@ class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
RoleInstance ri = allocations[0]
assert ri.role == dynamic
- assert ri.roleId == priority_num_8
+ assert ri.roleId == role_priority_8
assert ri.host.host == targetHostname
// now look at the role history
def roleHistory = appState.roleHistory
- def activeNodes = roleHistory.listActiveNodes(priority_num_8)
+ def activeNodes = roleHistory.listActiveNodes(role_priority_8)
assert activeNodes.size() == 1
NodeInstance activeNode = activeNodes[0]
+ assert activeNode.get(role_priority_8)
+ def entry8 = activeNode.get(role_priority_8)
+ assert entry8.active == 1
assert activeNode.hostname == targetHostname
-
- // now trigger a termination event on that role
+
+ def activeNodeInstance = roleHistory.getOrCreateNodeInstance(ri.container)
+
+ assert activeNode == activeNodeInstance
+ def entry
+ assert (entry = activeNodeInstance.get(role_priority_8)) != null
+ assert entry.active
+ assert entry.live
+ // now trigger a termination event on that role
+
+ // increment time for a long-lived failure event
+ appState.time = appState.time + 100000
+
+ log.debug("Triggering failure")
def cid = ri.id
- // failure
AppState.NodeCompletionResult result = appState.onCompletedNode(
containerStatus(cid, 1))
assert result.roleInstance == ri
assert result.containerFailed
+
+ roleHistory.dump();
+ // values should have changed
+ assert entry.failed == 1
+ assert !entry.startFailed
+ assert !entry.active
+ assert !entry.live
+
- def nodeForNewInstance = roleHistory.findNodeForNewInstance(
- dynamicRoleStatus)
- assert nodeForNewInstance
+ def nodesForRoleId = roleHistory.getNodesForRoleId(role_priority_8)
+ assert nodesForRoleId
// make sure new nodes will default to a different host in the engine
assert targetNode < engine.allocator.nextIndex()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b06d631/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index ad85b89..e683587 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -30,6 +30,8 @@ class MockAppState extends AppState {
super(recordFactory);
}
+ long time = 0;
+
/**
* Instance with a mock record factory
*/
@@ -40,4 +42,13 @@ class MockAppState extends AppState {
public Map<String, ProviderRole> getRoleMap() {
return super.roleMap;
}
+
+ /**
+ * Current time. if the <code>time</code> field
+ * is set, that value is returned
+ * @return the current time.
+ */
+ protected long now() {
+ return time ?: System.currentTimeMillis();
+ }
}
[46/50] git commit: SLIDER-575. Slider fun-test stabilization (HBase
fun tests)
Posted by st...@apache.org.
SLIDER-575. Slider fun-test stabilization (HBase fun tests)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0192716
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0192716
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0192716
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: b019271621530cd2dde2f0a44a557d8ee8e52c42
Parents: b072b9f
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Nov 1 14:05:27 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Nov 1 14:05:27 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/package/scripts/hbase_service.py | 4 ++--
app-packages/hbase/resources-default.json | 1 +
.../main/java/org/apache/slider/common/tools/CoreFileSystem.java | 4 ++++
3 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0192716/app-packages/hbase/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index 96add84..db663b8 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -34,7 +34,7 @@ def hbase_service(
no_op_test = None
if action == 'start':
- daemon_cmd = format("{cmd} start {role}")
+ daemon_cmd = format("env HBASE_IDENT_STRING={hbase_user} {cmd} start {role}")
if name == 'rest':
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
elif name == 'thrift':
@@ -43,7 +43,7 @@ def hbase_service(
daemon_cmd = format("{daemon_cmd} -p {thrift2_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
- daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
+ daemon_cmd = format("env HBASE_IDENT_STRING={hbase_user} {cmd} stop {role} && rm -f {pid_file}")
if daemon_cmd is not None:
Execute ( daemon_cmd,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0192716/app-packages/hbase/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources-default.json b/app-packages/hbase/resources-default.json
index ff58989..b1da1f7 100644
--- a/app-packages/hbase/resources-default.json
+++ b/app-packages/hbase/resources-default.json
@@ -13,6 +13,7 @@
"yarn.memory": "1500"
},
"slider-appmaster": {
+ "yarn.memory": "1024"
},
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0192716/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 7377dd3..fed0bc2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -563,6 +563,7 @@ public class CoreFileSystem {
public Map<String, Path> listPersistentInstances() throws IOException {
FileSystem fs = getFileSystem();
Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
+ log.debug("Looking for all persisted application at {}", path.toString());
if (!fs.exists(path)) {
// special case: no instances have ever been created
return new HashMap<String, Path>(0);
@@ -579,6 +580,9 @@ public class CoreFileSystem {
if (fs.exists(internalJson)) {
// success => this is an instance
instances.put(child.getName(), child);
+ } else {
+ log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.",
+ child.toString());
}
}
}
[38/50] git commit: Merge branch
'feature/SLIDER-587-dynamic_role_placement' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-587-dynamic_role_placement' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f83ce757
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f83ce757
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f83ce757
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: f83ce7571aeafcf782d6462ef779479c1f08924c
Parents: a9e81c0 0b06d63
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 21:24:32 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 21:24:32 2014 +0000
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 37 ++--
.../server/appmaster/state/NodeEntry.java | 12 +-
.../server/appmaster/state/NodeInstance.java | 4 +-
.../server/appmaster/state/RoleHistory.java | 52 +++--
.../server/appmaster/state/RoleInstance.java | 5 +
.../TestMockAppStateContainerFailure.groovy | 7 +-
.../TestMockAppStateDynamicHistory.groovy | 211 +++++++++++++++++++
.../TestMockAppStateDynamicRoles.groovy | 14 +-
.../TestMockAppStateFlexDynamicRoles.groovy | 5 +-
.../TestMockAppStateRMOperations.groovy | 3 +-
.../TestMockAppStateRebuildOnAMRestart.groovy | 10 +-
.../TestMockAppStateRolePlacement.groovy | 4 +-
.../appmaster/model/mock/Allocator.groovy | 8 +-
.../model/mock/BaseMockAppStateTest.groovy | 36 +++-
.../appmaster/model/mock/MockAppState.groovy | 25 +++
15 files changed, 364 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
[19/50] git commit: SLIDER-263 AM no longer persists keystore password
Posted by st...@apache.org.
SLIDER-263 AM no longer persists keystore password
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1a071e31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1a071e31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1a071e31
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: 1a071e31dd2f2c4486bcbdfe2f02065efe636545
Parents: d03bf28
Author: Jon Maron <jm...@hortonworks.com>
Authored: Thu Oct 30 21:13:37 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Thu Oct 30 21:13:37 2014 -0400
----------------------------------------------------------------------
.../org/apache/slider/common/SliderKeys.java | 1 -
.../server/appmaster/SliderAppMaster.java | 11 +++---
.../services/security/CertificateManager.java | 4 +--
.../server/services/security/SecurityUtils.java | 35 ++++++++++++--------
.../agent/TestAgentAMManagementWS.groovy | 16 ++++++---
.../web/rest/agent/TestAMAgentWebServices.java | 6 ++--
6 files changed, 42 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a071e31/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 89cc263..f11d200 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -174,7 +174,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String KEY_FILE_NAME = "ca.key";
String KEYSTORE_FILE_NAME = "keystore.p12";
String CRT_PASS_FILE_NAME = "pass.txt";
- String PASSPHRASE = "DEV";
String PASS_LEN = "50";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a071e31/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index efa1b09..39a2572 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -687,14 +687,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Start up the WebApp and track the URL for it
certificateManager = new CertificateManager();
- certificateManager.initialize(
- instanceDefinition.getAppConfOperations()
- .getComponent(SliderKeys.COMPONENT_AM));
+ MapOperations component = instanceDefinition.getAppConfOperations()
+ .getComponent(SliderKeys.COMPONENT_AM);
+ certificateManager.initialize(component);
certificateManager.setPassphrase(instanceDefinition.getPassphrase());
- if (instanceDefinition.
- getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM).
- getOptionBool(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
+ if (component.getOptionBool(
+ AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
uploadServerCertForLocalization(clustername, fs);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a071e31/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index 3771962..257f8f9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -61,8 +61,8 @@ public class CertificateManager {
private String passphrase;
/**
- * Verify that root certificate exists, generate it otherwise.
- */
+ * Verify that root certificate exists, generate it otherwise.
+ */
public void initialize(MapOperations compOperations) {
SecurityUtils.initializeSecurityParameters(compOperations);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a071e31/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
index c7ad8dd..527d4e6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -140,6 +140,11 @@ public class SecurityUtils {
}
public static void initializeSecurityParameters(MapOperations configMap) {
+ initializeSecurityParameters(configMap, false);
+ }
+
+ public static void initializeSecurityParameters(MapOperations configMap,
+ boolean persistPassword) {
String keyStoreLocation = configMap.getOption(
SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
File secDirFile = new File(keyStoreLocation).getParentFile();
@@ -167,26 +172,28 @@ public class SecurityUtils {
}
// need to create the password
}
- keystorePass = getKeystorePassword(secDirFile);
+ keystorePass = getKeystorePassword(secDirFile, persistPassword);
securityDir = secDirFile.getAbsolutePath();
}
- private static String getKeystorePassword(File secDirFile) {
+ private static String getKeystorePassword(File secDirFile,
+ boolean persistPassword) {
File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME);
String password = null;
-
if (!passFile.exists()) {
- LOG.info("Generation of file with password");
- try {
- password = RandomStringUtils.randomAlphanumeric(
- Integer.valueOf(SliderKeys.PASS_LEN));
- FileUtils.writeStringToFile(passFile, password);
- passFile.setWritable(true);
- passFile.setReadable(true);
- } catch (IOException e) {
- e.printStackTrace();
- throw new RuntimeException(
- "Error creating certificate password file");
+ LOG.info("Generating keystore password");
+ password = RandomStringUtils.randomAlphanumeric(
+ Integer.valueOf(SliderKeys.PASS_LEN));
+ if (persistPassword) {
+ try {
+ FileUtils.writeStringToFile(passFile, password);
+ passFile.setWritable(true);
+ passFile.setReadable(true);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(
+ "Error creating certificate password file");
+ }
}
} else {
LOG.info("Reading password from existing file");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a071e31/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index a6c351d..7434c4e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -23,6 +23,7 @@ import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.exceptions.YarnException
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
@@ -57,6 +58,7 @@ import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
@CompileStatic
@Slf4j
class TestAgentAMManagementWS extends AgentTestBase {
+ private static String password;
public static final String AGENT_URI = "ws/v1/slider/agents/";
final static Logger logger = LoggerFactory.getLogger(TestAgentAMManagementWS.class)
@@ -91,11 +93,11 @@ class TestAgentAMManagementWS extends AgentTestBase {
super.setup()
MapOperations compOperations = new MapOperations();
compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, "/tmp/work/security/keystore.p12");
- SecurityUtils.initializeSecurityParameters(compOperations);
+ SecurityUtils.initializeSecurityParameters(compOperations, true);
CertificateManager certificateManager = new CertificateManager();
certificateManager.initialize(compOperations);
String keystoreFile = SecurityUtils.getSecurityDir() + File.separator + SliderKeys.KEYSTORE_FILE_NAME;
- String password = SecurityUtils.getKeystorePass();
+ password = SecurityUtils.getKeystorePass();
System.setProperty("javax.net.ssl.trustStore", keystoreFile);
System.setProperty("javax.net.ssl.trustStorePassword", password);
System.setProperty("javax.net.ssl.trustStoreType", "PKCS12");
@@ -184,9 +186,11 @@ class TestAgentAMManagementWS extends AgentTestBase {
InstanceBuilder builder)
throws IOException, SliderException, LockAcquireFailedException {
AggregateConf conf = builder.getInstanceDescription()
- conf.getAppConfOperations().getComponent("slider-appmaster").put(
+ MapOperations component = conf.getAppConfOperations().getComponent("slider-appmaster")
+ component.put(
"ssl.server.keystore.location",
"/tmp/work/security/keystore.p12")
+ component.put("ssl.server.keystore.password", password)
super.persistInstanceDefinition(overwrite, appconfdir, builder)
}
@@ -196,10 +200,12 @@ class TestAgentAMManagementWS extends AgentTestBase {
AggregateConf instanceDefinition,
boolean debugAM)
throws YarnException, IOException {
- instanceDefinition.getAppConfOperations().getComponent("slider-appmaster").put(
+ MapOperations component = instanceDefinition.getAppConfOperations().getComponent("slider-appmaster")
+ component.put(
"ssl.server.keystore.location",
"/tmp/work/security/keystore.p12")
- return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM)
+ component.put("ssl.server.keystore.password", password)
+ return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1a071e31/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index daa47fa..faec5d8 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -80,10 +80,10 @@ public class TestAMAgentWebServices {
}
});
- SecurityUtils.initializeSecurityParameters(new MapOperations());
- MapOperations compOperations = new MapOperations();
+ MapOperations configMap = new MapOperations();
+ SecurityUtils.initializeSecurityParameters(configMap, true);
CertificateManager certificateManager = new CertificateManager();
- certificateManager.initialize(compOperations);
+ certificateManager.initialize(configMap);
String keystoreFile = SecurityUtils.getSecurityDir() + File.separator + SliderKeys.KEYSTORE_FILE_NAME;
String password = SecurityUtils.getKeystorePass();
System.setProperty("javax.net.ssl.trustStore", keystoreFile);
[31/50] git commit: SLIDER-570: tuning of tests,
agent failure tests working (i.e. regression fixed)
Posted by st...@apache.org.
SLIDER-570: tuning of tests, agent failure tests working (i.e. regression fixed)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1023ead
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1023ead
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1023ead
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: e1023eadf896866ce6c63f06e47bf835adc8576c
Parents: 6e171c7
Author: Steve Loughran <st...@apache.org>
Authored: Fri Oct 31 13:25:26 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Oct 31 13:25:26 2014 +0000
----------------------------------------------------------------------
.../framework/AgentCommandTestBase.groovy | 10 ++++
.../funtest/framework/CommandTestBase.groovy | 54 ++++++++++++--------
.../funtest/lifecycle/AMFailuresIT.groovy | 3 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 1 +
.../funtest/lifecycle/AgentFailures2IT.groovy | 12 ++---
.../funtest/lifecycle/AgentFailuresIT.groovy | 8 +--
.../lifecycle/AgentLaunchFailureIT.groovy | 3 +-
.../funtest/lifecycle/AgentRegistryIT.groovy | 2 -
.../lifecycle/AppsThroughAgentDemo.groovy | 27 ++++++----
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 16 +++---
.../AppsThroughAgentQueueAndLabelsIT.groovy | 6 ++-
11 files changed, 84 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 6c99ab6..6e05056 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -167,4 +168,13 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assert fail("Old cluster either should not exist or should get destroyed; destroy exit code = ${shell.ret}")
}
}
+
+ /**
+ * Assert that the application is running (i.e in state
+ * {@link YarnApplicationState#RUNNING})
+ * @param appId application ID
+ */
+ def assertAppRunning(String appId) {
+ assertInYarnState(appId, YarnApplicationState.RUNNING)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index fc74129..7ea84df 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -94,8 +94,7 @@ abstract class CommandTestBase extends SliderTestUtils {
public static final int CONTAINER_LAUNCH_TIMEOUT = 90000
public static final int PROBE_SLEEP_TIME = 4000
public static final int REGISTRY_STARTUP_TIMEOUT = 60000
- public static
- final String E_LAUNCH_FAIL = 'Application did not start'
+ public static final String E_LAUNCH_FAIL = 'Application did not start'
/*
Static initializer for test configurations. If this code throws exceptions
@@ -380,14 +379,18 @@ abstract class CommandTestBase extends SliderTestUtils {
static SliderShell lookup(int result, String id, File out) {
assert id
def commands = [ACTION_LOOKUP, ARG_ID, id]
- if (out) commands += [ARG_OUTPUT, out.absolutePath]
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
+ }
slider(result, commands)
}
static SliderShell lookup(String id, File out) {
assert id
def commands = [ACTION_LOOKUP, ARG_ID, id]
- if (out) commands += [ARG_OUTPUT, out.absolutePath]
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
+ }
slider(commands)
}
@@ -707,7 +710,7 @@ abstract class CommandTestBase extends SliderTestUtils {
}
return args
}
-
+
public static SerializedApplicationReport maybeLoadAppReport(File reportFile) {
if (reportFile.exists() && reportFile.length()> 0) {
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
@@ -776,19 +779,24 @@ abstract class CommandTestBase extends SliderTestUtils {
ARG_MESSAGE, "suicide"
])
-
sleep(5000)
ensureApplicationIsUp(cluster)
-
-/*
- def sleeptime = SLIDER_CONFIG.getInt(KEY_AM_RESTART_SLEEP_TIME,
- DEFAULT_AM_RESTART_SLEEP_TIME)
- sleep(sleeptime)
-*/
- ClusterDescription status
+ return sliderClient.clusterDescription
+ }
+ public ClusterDescription killAmAndWaitForRestart(
+ SliderClient sliderClient, String cluster, String appId) {
+
+ assert cluster
+ slider(0, [
+ ACTION_AM_SUICIDE, cluster,
+ ARG_EXITCODE, "1",
+ ARG_WAIT, "1000",
+ ARG_MESSAGE, "suicide"
+ ])
- status = sliderClient.clusterDescription
- return status
+ sleep(5000)
+ ensureYarnApplicationIsUp(appId)
+ return sliderClient.clusterDescription
}
protected void ensureRegistryCallSucceeds(String application) {
@@ -844,7 +852,6 @@ abstract class CommandTestBase extends SliderTestUtils {
);
}
-
/**
* is an application in a desired yarn state
* @param yarnState
@@ -1057,25 +1064,30 @@ abstract class CommandTestBase extends SliderTestUtils {
int expectedCount = args['limit'].toInteger();
int requestedCount = queryRequestedCount(application, role)
- log.debug("requested count = $requestedCount; expected=$expectedCount")
+ log.debug("requested $role count = $requestedCount; expected=$expectedCount")
return Outcome.fromBool(requestedCount >= expectedCount)
}
- void expectContainerRequestedCountReached(String application, String role, int limit) {
+ void expectContainerRequestedCountReached(String application, String role, int limit,
+ int container_launch_timeout) {
repeatUntilSuccess(
this.&hasRequestedContainerCountReached,
- CONTAINER_LAUNCH_TIMEOUT,
+ container_launch_timeout,
PROBE_SLEEP_TIME,
[limit : Integer.toString(limit),
role : role,
application: application],
true,
"countainer count not reached") {
- describe "container count not reached"
+ int requestedCount = queryRequestedCount(application, role)
+
+ def message = "expected count of $role = $limit not reached: $requestedCount" +
+ " after $container_launch_timeout mS"
+ describe message
ClusterDescription cd = execStatus(application);
log.info("Parsed status \n$cd")
- status(application).dumpOutput()
+ fail(message)
};
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index 2e28c84..7cc01b8 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -70,7 +70,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
def appId = ensureYarnApplicationIsUp(launchReportFile)
- expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
// Wait for 20 secs for AM and agent to both reach STARTED state
sleep(1000 * 20)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 3e5cec7..cb137ce 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -183,6 +183,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
//start with a restart count set to enable restart
describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
+
thaw(CLUSTER,
[
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 6c6b52b..bfae6ec 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -60,13 +59,12 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
def appId = ensureYarnApplicationIsUp(launchReportFile)
- expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+ CONTAINER_LAUNCH_TIMEOUT * 2)
sleep(1000 * 20)
- def cd = execStatus(APPLICATION_NAME)
- assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 3
- assertInYarnState(appId, YarnApplicationState.RUNNING)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3,
+ CONTAINER_LAUNCH_TIMEOUT * 2)
+ assertAppRunning(appId)
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 0410881..efd6194 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -63,13 +63,13 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
logShell(shell)
def appId = ensureYarnApplicationIsUp(launchReportFile)
-
- expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 2,
+ CONTAINER_LAUNCH_TIMEOUT)
sleep(1000 * 20)
- assertInYarnState(appId, YarnApplicationState.RUNNING)
+ assertAppRunning(appId)
def cd = expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
assert cd.statistics[COMMAND_LOGGER]["containers.requested"] >= 2
- assertInYarnState(appId, YarnApplicationState.RUNNING)
+ assertAppRunning(appId)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
index 5087c06..791123c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT.groovy
@@ -28,7 +28,6 @@ import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -62,7 +61,7 @@ public class AgentLaunchFailureIT extends AgentCommandTestBase
// create an AM which fails to launch
File launchReportFile = createAppReportFile();
- SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+ createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
[
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index b5ee23d..5b8b3cd 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -132,7 +132,5 @@ public class AgentRegistryIT extends AgentCommandTestBase
//cluster now missing
exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
-
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
index eb825fc..5be7211 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
@@ -49,13 +49,17 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
+ File launchReportFile = createAppReportFile();
+
SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME,
APP_TEMPLATE,
- APP_RESOURCE)
+ APP_RESOURCE,
+ [],
+ launchReportFile)
logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
//flex
slider(EXIT_SUCCESS,
@@ -69,9 +73,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// sleep till the new instance starts
sleep(1000 * 10)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
+ slider(EXIT_SUCCESS,
+ [ACTION_STATUS,
APPLICATION_NAME])
expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
@@ -88,7 +91,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ACTION_REGISTRY,
ARG_NAME,
APPLICATION_NAME,
- ARG_LISTEXP])
+ ARG_LISTEXP
+ ])
if(!containsString(shell, "container_log_dirs") ||
!containsString(shell, "container_work_dirs")) {
logShell(shell)
@@ -102,7 +106,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ARG_NAME,
APPLICATION_NAME,
ARG_GETEXP,
- "container_log_dirs"])
+ "container_log_dirs"
+ ])
if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
|| !containsString(shell, "\"level\" : \"component\"", 2)) {
logShell(shell)
@@ -116,7 +121,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ARG_NAME,
APPLICATION_NAME,
ARG_GETEXP,
- "container_work_dirs"])
+ "container_work_dirs"
+ ])
if(!containsString(shell, "\"tag\" : \"COMMAND_LOGGER\"", 2)
|| !containsString(shell, "\"level\" : \"component\"", 2)) {
logShell(shell)
@@ -132,7 +138,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
ARG_GETCONF,
"cl-site",
ARG_FORMAT,
- "json"])
+ "json"
+ ])
for (int i = 0; i < 10; i++) {
if (shell.getRet() != EXIT_SUCCESS) {
@@ -155,6 +162,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assert fail("Should have exported cl-site")
}
- assert isApplicationUp(APPLICATION_NAME), 'App is not running.'
+ assertAppRunning(appId)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 5a5b964..75807c3 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -76,10 +75,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// sleep till the new instance starts
sleep(1000 * 10)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
+ status(0, APPLICATION_NAME)
expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 2)
@@ -145,7 +141,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
if (shell.getRet() != EXIT_SUCCESS) {
println "Waiting for the cl-site to show up"
sleep(1000 * 10)
- shell = slider(
+ shell = slider(0,
[
ACTION_REGISTRY,
ARG_NAME,
@@ -156,12 +152,14 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
"json"])
}
}
- assert shell.getRet() == EXIT_SUCCESS, "cl-site should be retrieved"
if (!containsString(shell, "\"pattern.for.test.to.verify\" : \"verify this pattern\"", 1)) {
logShell(shell)
- assert fail("Should have exported cl-site")
+
+ fail("Should have exported cl-site; got " +
+ "stdout" +shell.stdErrHistory +
+ " \nstderr:" + shell.stdErrHistory)
}
- assertInYarnState(appId, YarnApplicationState.RUNNING)
+ assertAppRunning(appId)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1023ead/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index ee418dc..f6a1b1e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -92,7 +92,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
def appId = ensureYarnApplicationIsUp(launchReportFile)
- expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1 )
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
expectContainersLive(APPLICATION_NAME, COMMAND_LOGGER, 1)
//flex
@@ -107,7 +108,8 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
// spin till the flexed instance starts
ensureYarnApplicationIsUp(appId)
- expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3)
+ expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 3,
+ CONTAINER_LAUNCH_TIMEOUT)
sleep(1000 * 20)
[13/50] git commit: SLIDER-581 moving where the exit codes are set
up/using non superclass exec
Posted by st...@apache.org.
SLIDER-581 moving where the exit codes are set up/using non superclass exec
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ef516d9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ef516d9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ef516d9c
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: ef516d9cd704beb84fda08df4b510ca58c906c40
Parents: 518d750
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 30 14:37:10 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 30 14:37:10 2014 +0000
----------------------------------------------------------------------
.../org/apache/slider/funtest/commands/CommandExitCodesIT.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ef516d9c/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
index c49c738..73912e6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
@@ -36,7 +36,7 @@ public class CommandExitCodesIT extends CommandTestBase {
@Test
public void testStopWithoutCluster() throws Throwable {
- slider(LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, ["stop"])
+ slider(LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR, ["stop"]).dumpOutput()
}
[44/50] git commit: Revert "Remove assertion which expected an
exception in
TestMockAppStateFlexDynamicRoles#testDynamicFlexAddRoleConflictingPriority"
Posted by st...@apache.org.
Revert "Remove assertion which expected an exception in TestMockAppStateFlexDynamicRoles#testDynamicFlexAddRoleConflictingPriority"
This reverts commit c552af880f8d66f63e4cefdde118045c4e4b4593.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d03b6780
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d03b6780
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d03b6780
Branch: refs/heads/feature/SLIDER-531-registry-enhancements
Commit: d03b6780885c9069292768aa822820a96bd15305
Parents: a07eaf0
Author: Steve Loughran <st...@apache.org>
Authored: Sat Nov 1 11:20:32 2014 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Nov 1 11:20:32 2014 +0000
----------------------------------------------------------------------
.../model/appstate/TestMockAppStateFlexDynamicRoles.groovy | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d03b6780/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index aee2413..8308a13 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -123,6 +123,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
try {
appState.updateResourceDefinitions(cd.confTree);
dumpClusterDescription("updated CD", appState.getClusterStatus())
+ fail("Expected an exception")
} catch (BadConfigException expected) {
}
}