You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/23 18:20:32 UTC
[20/50] incubator-slider git commit: SLIDER-985: regression,
TestBuildBasicAgent failing. (cause: increasing the limit on the #of
agents allowed)
SLIDER-985: regression, TestBuildBasicAgent failing. (cause: increasing the limit on the #of agents allowed)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c776b1ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c776b1ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c776b1ad
Branch: refs/heads/develop
Commit: c776b1ad45dbd5403e2f6b2824254ba0f0935c5d
Parents: 7278c39
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 16 14:01:35 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 16 14:01:50 2015 +0000
----------------------------------------------------------------------
.../slider/core/persist/ConfPersister.java | 6 ++-
.../slider/providers/agent/AgentTestBase.groovy | 16 ++----
.../providers/agent/TestAgentAAEcho.groovy | 34 +++++++++---
.../providers/agent/TestBuildBasicAgent.groovy | 57 +++++++++++---------
slider-core/src/test/python/metainfo.xml | 2 +-
5 files changed, 68 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
index 60717f6..9759205 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
@@ -101,7 +101,7 @@ public class ConfPersister {
* Make the persistent directory
* @throws IOException IO failure
*/
- private void mkPersistDir() throws IOException {
+ public void mkPersistDir() throws IOException {
coreFS.getFileSystem().mkdirs(persistDir);
}
@@ -165,13 +165,15 @@ public class ConfPersister {
* Acquire the writelock
* @throws IOException IO
* @throws LockAcquireFailedException
- * @throws FileNotFoundException if the target dir does not exist
+ * @throws FileNotFoundException if the target dir does not exist.
*/
@VisibleForTesting
boolean acquireReadLock() throws FileNotFoundException,
IOException,
LockAcquireFailedException {
if (!coreFS.getFileSystem().exists(persistDir)) {
+ // the dir is not there, so the data is not there, so there
+ // is nothing to read
throw new FileNotFoundException(persistDir.toString());
}
long now = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 5bf1c1f..0e5cd00 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -32,6 +32,7 @@ import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
import org.junit.Rule
import org.junit.rules.TemporaryFolder
+import org.slf4j.Logger
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
@@ -53,7 +54,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
*/
public static void assumeValidServerEnv() {
try {
- SliderUtils.validateSliderServerEnvironment(log, true)
+ SliderUtils.validateSliderServerEnvironment(log as Logger, true)
} catch (Exception e) {
skip(e.toString())
}
@@ -135,12 +136,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
boolean create,
boolean blockUntilRunning) {
-
- YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
create ? SliderActions.ACTION_CREATE : SliderActions.ACTION_BUILD,
@@ -166,11 +162,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
List<String> extraArgs,
boolean deleteExistingData) {
- YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
SliderActions.ACTION_UPDATE,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
index 0b89f47..80ff5a8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -52,9 +52,7 @@ class TestAgentAAEcho extends TestAgentEcho {
validatePaths()
def echo = "echo"
- Map<String, Integer> roles = [
- (echo): 2,
- ];
+ Map<String, Integer> roles = buildRoleMap(echo)
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
roles,
[
@@ -74,10 +72,33 @@ class TestAgentAAEcho extends TestAgentEcho {
],
true, true,
true)
- SliderClient sliderClient = launcher.service
+ postLaunchActions(launcher.service, clustername, echo, roles)
+
+ }
+ /**
+ * Build the role map to use when creating teh cluster
+ * @param roleName the name used for the echo role
+ * @return the map
+ */
+ protected Map<String, Integer> buildRoleMap(String roleName) {
+ [
+ (roleName): 2,
+ ];
+ }
- def onlyOneEcho = [(echo): 1]
+ /**
+ * Any actions to perform after starting the agent cluster
+ * @param sliderClient client for the cluster
+ * @param clustername cluster name
+ * @param roleName name of the echo role
+ * @parm original set of roles
+ */
+ protected void postLaunchActions(SliderClient sliderClient,
+ String clustername,
+ String roleName,
+ Map<String, Integer> roles) {
+ def onlyOneEcho = [(roleName): 1]
waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME)
//sleep a bit
sleep(5000)
@@ -91,9 +112,8 @@ class TestAgentAAEcho extends TestAgentEcho {
sliderClient.flex(clustername, onlyOneEcho);
// while running, flex it with no changes
- sliderClient.flex(clustername, [(echo): 3]);
+ sliderClient.flex(clustername, [(roleName): 3]);
sleep(1000)
waitForRoleCount(sliderClient, onlyOneEcho, 1000)
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 264d260..60e9035 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -94,7 +94,9 @@ class TestBuildBasicAgent extends AgentTestBase {
1,
true,
false)
- buildAgentCluster("test_build_basic_agent_node_only",
+
+ def cluster01 = clustername + "_01"
+ buildAgentCluster(cluster01,
[(ROLE_NODE): 1],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -110,7 +112,9 @@ class TestBuildBasicAgent extends AgentTestBase {
def master = "hbase-master"
def rs = "hbase-rs"
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+
+ def cluster02 = clustername + "_02"
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(cluster02,
[
(ROLE_NODE): 1,
(master): 1,
@@ -149,10 +153,11 @@ class TestBuildBasicAgent extends AgentTestBase {
def rscomponent = resource.getMandatoryComponent(rs)
assert "5" == rscomponent.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
- // now create an instance with no role priority for the newnode role
+ describe "build a cluster with no role priority for the newnode role"
+
try {
- def name2 = clustername + "-2"
- buildAgentCluster(name2,
+ def cluster03 = clustername + "_03"
+ buildAgentCluster(cluster03,
[
(ROLE_NODE): 2,
"role3": 1,
@@ -166,14 +171,16 @@ class TestBuildBasicAgent extends AgentTestBase {
],
true, false,
false)
- failWithBuildSucceeding(name2, "no priority for one role")
+ failWithBuildSucceeding(cluster03, "no priority for one role")
} catch (BadConfigException expected) {
}
+ describe "build a cluster with the number of agents out of range"
try {
- launcher = buildAgentCluster(clustername + "-10",
+ def cluster04 = clustername + "_04"
+ launcher = buildAgentCluster(cluster04,
[
- (ROLE_NODE): 4,
+ (ROLE_NODE): 2000,
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -185,15 +192,18 @@ class TestBuildBasicAgent extends AgentTestBase {
],
true, false,
false)
- failWithBuildSucceeding(ROLE_NODE, "too many instances")
+ failWithBuildSucceeding(cluster04, "too many instances")
} catch (BadConfigException expected) {
- assert expected.message.contains("Expected minimum is 1 and maximum is 2")
- assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Expected minimum is 1 and maximum is")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "out of range.")
}
- //duplicate priorities
+
+ describe "build a cluster with duplicate priorities for roles"
try {
- def name3 = clustername + "-3"
- buildAgentCluster(name3,
+ def cluster05 = clustername + "_05"
+ buildAgentCluster(cluster05,
[
(ROLE_NODE): 5,
(master): 1,
@@ -206,16 +216,16 @@ class TestBuildBasicAgent extends AgentTestBase {
true, false,
false)
- failWithBuildSucceeding(name3, "duplicate priorities")
+ failWithBuildSucceeding(cluster05, "duplicate priorities")
} catch (BadConfigException expected) {
}
- def cluster4 = clustername + "-4"
+ def cluster06 = clustername + "_06"
def jvmopts = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
- buildAgentCluster(cluster4,
+ buildAgentCluster(cluster06,
[
(master): 1,
(rs): 5
@@ -235,7 +245,7 @@ class TestBuildBasicAgent extends AgentTestBase {
false)
//now we want to look at the value
- AggregateConf instanceDefinition = loadInstanceDefinition(cluster4)
+ AggregateConf instanceDefinition = loadInstanceDefinition(cluster06)
def opt = instanceDefinition.getAppConfOperations().getComponentOpt(
SliderKeys.COMPONENT_AM,
RoleKeys.JVM_OPTS,
@@ -245,8 +255,8 @@ class TestBuildBasicAgent extends AgentTestBase {
// now create an instance with no component options, hence no
// entry in the app config
- def name5 = clustername + "-5"
- buildAgentCluster(name5,
+ def name07 = clustername + "_07"
+ buildAgentCluster(name07,
[
"hbase-rs": 1,
],
@@ -403,9 +413,8 @@ class TestBuildBasicAgent extends AgentTestBase {
def rscomponent2 = resource2.getMandatoryComponent(rs)
assert "6" == rscomponent2.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
}
-
+
public AggregateConf loadInstanceDefinition(String name) {
- def cluster4
def sliderFS = createSliderFileSystem()
def dirPath = sliderFS.buildClusterDirPath(name)
ConfPersister persister = new ConfPersister(sliderFS, dirPath)
@@ -697,10 +706,8 @@ class TestBuildBasicAgent extends AgentTestBase {
}
public void failWithBuildSucceeding(String name, String reason) {
- def badArgs1
AggregateConf instanceDefinition = loadInstanceDefinition(name)
- log.error(
- "Build operation should have failed from $reason : \n$instanceDefinition")
+ log.error("Build operation should have failed from $reason : \n$instanceDefinition")
fail("Build operation should have failed from $reason")
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c776b1ad/slider-core/src/test/python/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index 2a8c9e0..7f9cd23 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -51,7 +51,7 @@
<name>echo</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>200</maxInstanceCount>
+ <maxInstanceCount>100</maxInstanceCount>
<commandScript>
<script>echo.py</script>
<scriptType>PYTHON</scriptType>