You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/10/03 00:26:03 UTC
[6/7] git commit: SLIDER-470 slider appears to support negative
component counts: flex checks and in AM (not tested that)
SLIDER-470 slider appears to support negative component counts: flex checks and in AM (not tested that)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cbf3a634
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cbf3a634
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cbf3a634
Branch: refs/heads/develop
Commit: cbf3a634b82be14c714d05d59b10ee9959fe08f6
Parents: 483ab95
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 2 14:59:02 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 2 14:59:02 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 18 ++--
.../apache/slider/core/conf/AggregateConf.java | 20 +++++
.../apache/slider/providers/ProviderCore.java | 4 +-
.../TestFreezeThawFlexStandaloneAM.groovy | 94 ++++++++++++++++++++
.../TestFreezeThawMasterlessAM.groovy | 92 -------------------
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
...ndaloneClusterNegativeComponentCounts.groovy | 79 ----------------
.../slider/providers/agent/TestAgentEcho.groovy | 18 +++-
8 files changed, 140 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 05ef532..dc81f98 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2053,29 +2053,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
clusterDirectory);
- SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
- // provider to validate what there is
- SliderAMClientProvider provider = sliderAM;
- validateInstanceDefinition(provider, instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- if (count < 0) {
- throw new BadCommandArgumentsException("Requested number of " + role
- + " instances is out of range");
- }
resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
Integer.toString(count));
-
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
role,
count,
resources);
}
+ SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+ // slider provider to validate what there is
+ validateInstanceDefinition(sliderAM, instanceDefinition);
+
int exitCode = EXIT_FALSE;
// save the specification
try {
@@ -2083,10 +2078,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} catch (LockAcquireFailedException e) {
// lock failure
log.debug("Failed to lock dir {}", clusterDirectory, e);
- log.warn("Failed to save new resource definition to {} : {}", clusterDirectory,
- e.toString());
-
-
+ log.warn("Failed to save new resource definition to {} : {}", clusterDirectory, e);
}
// now see if it is actually running and tell it about the update if it is
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
index 02669cd..e9e0dc9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -23,6 +23,8 @@ import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
+import java.io.IOException;
+
/**
* Aggregate Configuration.
*
@@ -58,6 +60,24 @@ public final class AggregateConf {
setInternal(internal);
}
+ /**
+ * Take a snapshot of the configuration
+ * @param instanceDefinition source
+ * @throws IOException marshalling/copying problems
+ */
+ public AggregateConf(AggregateConf instanceDefinition) throws IOException {
+ ConfTreeOperations resourcesSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getResources());
+ ConfTreeOperations appConfSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
+ ConfTreeOperations internalsSnapshot =
+ ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
+ //build a new aggregate from the snapshots
+ setResources(resourcesSnapshot.confTree);
+ setAppConf(appConfSnapshot.confTree);
+ setInternal(internalsSnapshot.confTree);
+ }
+
public void setResources(ConfTree resources) {
this.resources = resources;
resourceOperations = new ConfTreeOperations(resources);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
index 65d2138..9767430 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.SliderException;
import java.util.List;
@@ -28,7 +29,7 @@ public interface ProviderCore {
String getName();
List<ProviderRole> getRoles();
-
+
Configuration getConf();
/**
@@ -38,4 +39,5 @@ public interface ProviderCore {
*/
void validateInstanceDefinition(AggregateConf instanceDefinition) throws
SliderException;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
new file mode 100644
index 0000000..7708502
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawFlexStandaloneAM.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.freezethaw
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * stop and start an AM
+ */
+@CompileStatic
+@Slf4j
+
+class TestFreezeThawFlexStandaloneAM extends AgentMiniClusterTestBase {
+
+ File getConfDirFile() {
+ return new File("target/testFreezeThawFlexStandaloneAM/conf")
+ }
+
+ @Override
+ String getConfDir() {
+ return confDirFile.toURI().toString()
+ }
+
+ @Test
+ public void testFreezeThawFlexStandaloneAM() throws Throwable {
+ YarnConfiguration conf = configuration
+ String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
+
+ describe "create a standalone AM, stop it, start it"
+ //copy the confdir somewhere
+ Path resConfPath = new Path(resourceConfDirURI)
+ Path tempConfPath = new Path(confDir)
+ SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null)
+
+
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+
+
+ // here we do something devious: delete our copy of the configuration
+ // this makes sure the remote config gets picked up
+ HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf)
+ localFS.delete(tempConfPath,true)
+
+ //now start the cluster
+ ServiceLauncher launcher2 = thawCluster(clustername, [], true);
+ SliderClient newCluster = launcher2.service
+ addToTeardown(newCluster);
+
+ newCluster.getClusterDescription(clustername);
+
+ // while running, flex it with no changes
+ newCluster.flex(clustername, [:]);
+
+ //stop
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+
+ //stop again
+ assert 0 == clusterActionFreeze(sliderClient, clustername)
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
deleted file mode 100644
index 5a1acef..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.freezethaw
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.FileSystem as HadoopFS
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * stop and start an AM
- */
-@CompileStatic
-@Slf4j
-
-class TestFreezeThawMasterlessAM extends AgentMiniClusterTestBase {
-
- File getConfDirFile() {
- return new File("target/TestFreezeThawMasterlessAM/conf")
- }
-
- @Override
- String getConfDir() {
- return confDirFile.toURI().toString()
- }
-
- @Test
- public void testFreezeThawMasterlessAM() throws Throwable {
- YarnConfiguration conf = configuration
- String clustername = createMiniCluster("", conf, 1, 1, 1, true, false)
-
- describe "create a masterless AM, stop it, start it"
- //copy the confdir somewhere
- Path resConfPath = new Path(resourceConfDirURI)
- Path tempConfPath = new Path(confDir)
- SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null)
-
-
- ServiceLauncher<SliderClient> launcher = createStandaloneAM(
- clustername,
- true,
- true)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- assert 0 == clusterActionFreeze(sliderClient, clustername)
-
-
- // here we do something devious: delete our copy of the configuration
- // this makes sure the remote config gets picked up
- HadoopFS localFS = HadoopFS.get(tempConfPath.toUri(), conf)
- localFS.delete(tempConfPath,true)
-
- //now start the cluster
- ServiceLauncher launcher2 = thawCluster(clustername, [], true);
- SliderClient newCluster = launcher2.service
- addToTeardown(newCluster);
-
-// ApplicationReport report = waitForClusterLive(newCluster)
- newCluster.getClusterDescription(clustername);
- //stop
- assert 0 == clusterActionFreeze(sliderClient, clustername)
-
- //stop again
- assert 0 == clusterActionFreeze(sliderClient, clustername)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 4b66e1b..5e8ac9e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -44,7 +44,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
@Test
public void testStandaloneAgentAM() throws Throwable {
- describe "create a masterless AM then get the service and look it up via the AM"
+ describe "create a masterless AM then perform actions on it"
//launch fake master
String clustername = createMiniCluster("", configuration, 1, true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
deleted file mode 100644
index 8a23dc5..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneClusterNegativeComponentCounts.groovy
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.api.InternalKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-@CompileStatic
-@Slf4j
-class TestStandaloneClusterNegativeComponentCounts
- extends AgentMiniClusterTestBase {
-
- @Test
- public void testStandaloneClusterNegativeComponentCounts() throws Throwable {
- describe "Run a Standalone AM with cluster counts set to 0 ... expect failures"
- // patch the configuration for AM restart
- int threshold = 2;
- YarnConfiguration conf = getRestartableConfiguration(threshold)
-
- String clustername = createMiniCluster("", conf, 1, true)
- if (hdfsCluster) {
- fail("Agent tests do not (currently) work with mini HDFS cluster")
- }
- ServiceLauncher<SliderClient> launcher =
- createCluster(clustername,
- ["negative":-4],
- [
- ],
- true,
- false,
- agentDefOptions)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- ApplicationReport report
- report = waitForClusterLive(sliderClient, 30000)
- }
-
- /**
- * Get a restartable configuration
- * @param restarts
- * @return
- */
- public YarnConfiguration getRestartableConfiguration(int restarts) {
- def conf = new YarnConfiguration(configuration)
- conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
- conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
- conf
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cbf3a634/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index a7b5fe3..f67ee92 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -23,6 +23,8 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
@@ -85,9 +87,23 @@ class TestAgentEcho extends AgentTestBase {
waitForRoleCount(sliderClient, roles, AGENT_CLUSTER_STARTUP_TIME)
//sleep a bit
- sleep(20000)
+ sleep(5000)
//expect the role count to be the same
waitForRoleCount(sliderClient, roles, 1000)
+ // flex size
+ // while running, flex it with no changes
+ sliderClient.flex(clustername, [(role): 2]);
+ sleep(1000)
+ waitForRoleCount(sliderClient, roles, 1000)
+
+ // flex to an illegal value
+ try {
+ sliderClient.flex(clustername, [(role): -1]);
+ fail("expected an exception")
+ } catch (BadClusterStateException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "")
+ }
+
}
}