You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/09/09 17:14:41 UTC
[01/47] git commit: SLIDER-386 fixed role-specific failure threshold
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry 6e883bf91 -> 5784c4ab5
SLIDER-386 fixed role-specific failure threshold
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7ecab238
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7ecab238
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7ecab238
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 7ecab238a919ae4013f23b259ab4eba317635485
Parents: 82cf1f0
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Sep 2 08:48:10 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue Sep 2 08:48:10 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/state/AppState.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ecab238/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 07976ef..8b70185 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1559,7 +1559,7 @@ public class AppState {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
+ ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
[27/47] git commit: SLIDER-377 tagging some test files as
@CompileStatic
Posted by st...@apache.org.
SLIDER-377 tagging some test files as @CompileStatic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f2965d8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f2965d8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f2965d8a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f2965d8a056fcf6d56a39f0841f1903048c95c29
Parents: 88d72dd
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 17:54:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 17:54:46 2014 +0100
----------------------------------------------------------------------
.../slider/common/tools/TestClientResourceRegistration.groovy | 3 +++
.../org/apache/slider/common/tools/TestConfigHelperHDFS.groovy | 3 +++
.../org/apache/slider/common/tools/TestMiscSliderUtils.groovy | 3 +++
3 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
index 7150b3c..e3bfa1b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestClientResourceRegistration.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestClientResourceRegistration {
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
index 0d21d6c..d78aa1b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
@@ -27,6 +28,8 @@ import org.apache.slider.test.YarnMiniClusterTestBase
import org.junit.Test
@Slf4j
+@CompileStatic
+
class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
//diabled for now;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f2965d8a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
index 24367a3..9042af5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestMiscSliderUtils.groovy
@@ -18,12 +18,15 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.slider.test.SliderTestBase
import org.junit.Test
+@CompileStatic
+
class TestMiscSliderUtils extends SliderTestBase {
[02/47] git commit: SLIDER-387 adjusted default accumulo configuration
Posted by st...@apache.org.
SLIDER-387 adjusted default accumulo configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0ac2536f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0ac2536f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0ac2536f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0ac2536f96e9010569deb0a0d2079f8c23c17e06
Parents: 7ecab23
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Sep 2 11:17:46 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue Sep 2 11:17:46 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 5 +++--
.../accumulo/src/test/resources/appConfig_monitor_ssl.json | 5 +++--
app-packages/accumulo/src/test/resources/appConfig_ssl.json | 5 +++--
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index a44dbad..2c0c2d5 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "128m",
+ "site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -27,12 +27,13 @@
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
index c6a2639..45e6e19 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "128m",
+ "site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -27,12 +27,13 @@
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ac2536f/app-packages/accumulo/src/test/resources/appConfig_ssl.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
index f48216d..3daaaaf 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
@@ -9,7 +9,7 @@
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "128m",
+ "site.global.tserver_heapsize": "256m",
"site.global.master_heapsize": "128m",
"site.global.monitor_heapsize": "64m",
"site.global.gc_heapsize": "64m",
@@ -29,12 +29,13 @@
"site.accumulo-site.instance.rpc.ssl.enabled": "true",
"site.accumulo-site.instance.rpc.ssl.clientAuth": "true",
"site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
"site.accumulo-site.tserver.memory.maps.max": "80M",
"site.accumulo-site.tserver.cache.data.size": "7M",
"site.accumulo-site.tserver.cache.index.size": "20M",
"site.accumulo-site.trace.user": "root",
"site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "100M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
"site.accumulo-site.master.port.client": "0",
"site.accumulo-site.trace.port.client": "0",
"site.accumulo-site.tserver.port.client": "0",
[41/47] git commit: SLIDER-397 -fix regression surfacing on jenkins
(editing wrong "kill" command)
Posted by st...@apache.org.
SLIDER-397 -fix regression surfacing on jenkins (editing wrong "kill" command)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2d6331a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2d6331a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2d6331a0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2d6331a0eca42fc19b40f741e31a1580ab8ae8b8
Parents: 56cd866
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 09:57:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 09:57:11 2014 +0100
----------------------------------------------------------------------
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d6331a0/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index dbf5c49..cbf4749 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -747,7 +747,7 @@ class SliderTestUtils extends Assert {
def commandString
if (!Shell.WINDOWS) {
- GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill /f $signal"
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
log.info("Command command = $killCommand" )
commandString = ["bash", "-c", killCommand]
[47/47] git commit: SLIDER-149 in sync with YARN-913-patch-002
Posted by st...@apache.org.
SLIDER-149 in sync with YARN-913-patch-002
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5784c4ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5784c4ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5784c4ab
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 5784c4ab5e2cae45625d2dfa745802c69d5f19ab
Parents: 901a673
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 9 16:12:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 9 16:12:46 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 7 +-
.../server/appmaster/SliderAppMaster.java | 18 +-
.../TestStandaloneYarnRegistryAM.groovy | 378 ++++++++++++++++++
.../agent/standalone/TestYarnRegistryAM.groovy | 379 -------------------
4 files changed, 392 insertions(+), 390 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8219fd7..ed3bc0e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -136,6 +136,7 @@ import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -2376,13 +2377,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException Network or other problems
*/
@VisibleForTesting
- public List<ServiceRecord> actionRegistryListYarn(
+ public Collection<ServiceRecord> actionRegistryListYarn(
ActionRegistryArgs registryArgs)
throws YarnException, IOException {
String serviceType = registryArgs.serviceType;
String name = registryArgs.name;
RegistryOperationsService operations = getRegistryOperations();
- List<ServiceRecord> serviceRecords;
+ Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
String serviceclassPath =
BindingUtils.serviceclassPath(BindingUtils.currentUser(),
@@ -2394,7 +2395,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ serviceclassPath);
}
serviceRecords =
- RecordOperations.extractServiceRecords(operations, listDir);
+ RecordOperations.extractServiceRecords(operations, listDir).values();
} else {
ServiceRecord instance = lookupServiceRecord(registryArgs);
serviceRecords = new ArrayList<ServiceRecord>(1);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9615b6f..4b5e0b3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -52,11 +52,12 @@ import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.yarn.registry.client.services.RegistryOperationsService;
import org.apache.hadoop.yarn.registry.client.types.PersistencePolicies;
import org.apache.hadoop.yarn.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.yarn.registry.server.services.ResourceManagerRegistryService;
+import org.apache.hadoop.yarn.registry.server.services.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -929,6 +930,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceRecord serviceRecord = new ServiceRecord();
String serviceID = appid.toString();
serviceRecord.id = serviceID;
+ serviceRecord.persistence = PersistencePolicies.APPLICATION;
serviceRecord.description = "Slider Application Master";
serviceRecord.addExternalEndpoint(
@@ -967,7 +969,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// and an ephemeral binding to the app
yarnRegistryOperations.putComponent(
- RegistryTypeUtils.yarnIdToDnsId(appAttemptID.toString()),
+ RegistryPathUtils.encodeYarnID(appAttemptID.toString()),
serviceRecord,
false);
@@ -982,10 +984,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
protected void setupInitialRegistryPaths() throws IOException {
- if (registryOperations instanceof ResourceManagerRegistryService) {
- ResourceManagerRegistryService rmRegOperations =
- (ResourceManagerRegistryService) registryOperations;
- rmRegOperations.createUserPath(service_user_name);
+ if (registryOperations instanceof RMRegistryOperationsService) {
+ RMRegistryOperationsService rmRegOperations =
+ (RMRegistryOperationsService) registryOperations;
+ rmRegOperations.createHomeDirectory(service_user_name);
}
}
@@ -1003,7 +1005,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
// this is where component registrations will go
log.info("Registering component {}", id);
- String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
+ String cid = RegistryPathUtils.encodeYarnID(id.toString());
ServiceRecord container = new ServiceRecord(
cid,
description,
@@ -1027,7 +1029,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
public void unregisterComponent(ContainerId id) {
log.info("Unregistering component {}", id);
- String cid = RegistryTypeUtils.yarnIdToDnsId(id.toString());
+ String cid = RegistryPathUtils.encodeYarnID(id.toString());
try {
yarnRegistryOperations.rmComponent(cid);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
new file mode 100644
index 0000000..c7291a8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.PathNotFoundException
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
+import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
+import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
+import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
+import org.apache.hadoop.yarn.registry.client.types.ServiceRecord
+
+import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
+import org.apache.slider.core.registry.docstore.PublishedConfigSet
+import org.apache.slider.core.registry.docstore.PublishedConfiguration
+import org.apache.slider.core.registry.docstore.UriMap
+import org.apache.slider.core.registry.info.CustomRegistryConstants
+import org.apache.slider.core.registry.retrieve.RegistryRetriever
+import org.apache.slider.server.appmaster.PublishedArtifacts
+import org.apache.slider.server.appmaster.web.rest.RestPaths
+import org.junit.Test
+
+/**
+ * work with a YARN registry
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
+
+
+ public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
+
+ @Test
+ public void testYarnRegistryAM() throws Throwable {
+
+
+ describe "create a masterless AM then perform YARN registry operations on it"
+
+
+ String clustername = createMiniCluster(configuration, 1, true)
+
+ // get local binding
+ def registryOperations = microZKCluster.registryOperations
+ registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
+
+ // verify the cluster has the YARN reg service live
+ def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry
+ assert rmRegistryService
+
+ ServiceLauncher<SliderClient> launcher
+ launcher = createStandaloneAM(clustername, true, false)
+ SliderClient client = launcher.service
+ addToTeardown(client);
+
+ ApplicationReport report = waitForClusterLive(client)
+ logReport(report)
+ List<ApplicationReport> apps = client.applications;
+
+ List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
+ SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)clusterNodes).size() == 1
+
+ ClusterNode masterNode = clusterNodes[0]
+ log.info("Master node = ${masterNode}");
+
+ List<ClusterNode> nodes
+ String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
+ assert uuids.length == 1;
+ nodes = client.listClusterNodes(uuids);
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ describe "AM Node UUID=${uuids[0]}"
+
+ nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ nodes = listNodesInRole(client, "")
+ assert ((List<ClusterNode>)nodes).size() == 1;
+ ClusterNode master = nodes[0]
+ assert master.role == SliderKeys.COMPONENT_AM
+
+
+
+
+ String username = client.username
+ def yarnRegistryClient = client.yarnAppListClient
+ describe("list of all applications")
+ logApplications(apps)
+ describe("apps of user $username")
+ List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
+ logApplications(userInstances)
+ assert userInstances.size() == 1
+ describe("named app $clustername")
+ ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
+ logReport(instance)
+ assert instance != null
+
+ // sleep to allow registration to complete
+ sleep(5000)
+
+
+
+
+ try {
+ def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
+ log.info("yarn service registry: \n${yarnRegistryDump}\n")
+ } catch (IOException ignored) {
+
+ }
+
+
+ describe "service registry names"
+ def registryService = client.registryOperations
+
+ def self = currentUser()
+ RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self))
+ dumpArray(serviceTypes)
+
+ def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
+
+ Map < String, ServiceRecord > recordMap = RecordOperations.extractServiceRecords(
+ registryService,
+ registryService.listDir(recordsPath))
+ def serviceRecords = recordMap.values();
+ dumpCollection(serviceRecords)
+ assert serviceRecords.size() == 1
+
+ def serviceInstance = serviceRecords[0]
+ log.info(serviceInstance.toString())
+
+ assert 2 <= serviceInstance.external.size()
+
+ // hit the registry web page
+
+ def registryEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.REGISTRY_REST_API)
+ assert registryEndpoint != null
+ def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
+ describe("Registry WADL @ $registryURL")
+
+ def publisherEndpoint = serviceInstance.getExternalEndpoint(
+ CustomRegistryConstants.PUBLISHER_REST_API)
+
+ def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
+ def publisher = publisherURL.toString()
+ describe("Publisher")
+
+ JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
+ def setlisting = GET(publisherURL)
+
+ log.info(setlisting)
+
+ UriMap uris = uriMapDeser.fromJson(setlisting)
+ assert uris.uris[RestPaths.SLIDER_CONFIGSET]
+ def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
+ JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
+ PublishedConfigSet)
+ def configSet = serDeser.fromJson(publishedJSON)
+ assert configSet.size() >= 1
+ assert configSet.contains(ARTIFACT_NAME)
+ PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
+
+ assert publishedYarnSite.empty
+
+ //get the full URL
+ def yarnSitePublisher = appendToURL(publisher,
+ RestPaths.SLIDER_CONFIGSET,
+ ARTIFACT_NAME)
+
+ String confJSON = GET(yarnSitePublisher)
+// log.info(confJSON)
+ JsonSerDeser< PublishedConfiguration> confSerDeser =
+ new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
+
+ publishedYarnSite = confSerDeser.fromJson(confJSON)
+
+ assert !publishedYarnSite.empty
+
+
+ //get the XML
+ def yarnSiteXML = yarnSitePublisher + ".xml"
+
+
+ String confXML = GET(yarnSiteXML)
+ log.info("Conf XML at $yarnSiteXML = \n $confXML")
+
+ String properties = GET(yarnSitePublisher + ".properties")
+ Properties parsedProps = new Properties()
+ parsedProps.load(new StringReader(properties))
+ assert parsedProps.size() > 0
+ def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
+ def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
+ assert rmAddrFromDownloadedProperties
+ assert rmHostnameFromDownloadedProperties
+
+ String json = GET(yarnSitePublisher + ".json")
+
+
+
+ describe("Registry List")
+ log.info(GET(registryURL))
+
+
+ describe "Registry Retrieval Class"
+ // retrieval
+
+ RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
+ log.info retriever.toString()
+
+ assert retriever.hasConfigurations(true)
+ PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
+ dumpConfigurationSet(externalConfSet)
+ assert externalConfSet[ARTIFACT_NAME]
+
+
+ describe "verify SLIDER-52 processing"
+ def yarnSite = retriever.retrieveConfiguration(
+ externalConfSet,
+ ARTIFACT_NAME,
+ true)
+ assert !yarnSite.empty
+ def siteXML = yarnSite.asConfiguration()
+ def rmHostnameViaClientSideXML = parsedProps.get(
+ YarnConfiguration.RM_HOSTNAME)
+ assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
+ def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
+
+ log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
+ assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
+
+ describe "fetch missing artifact"
+ try {
+ retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+ describe "Internal configurations"
+ assert !retriever.hasConfigurations(false)
+ try {
+ retriever.getConfigurations(false)
+ fail("expected a failure")
+ } catch (FileNotFoundException expected) {
+ // expected
+ }
+
+
+ // retrieval via API
+ ActionRegistryArgs registryArgs = new ActionRegistryArgs()
+ registryArgs.verbose = true
+
+ // list all
+ registryArgs.list = true;
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+
+ // list a named instance and expect a failure
+ registryArgs.list = true;
+ registryArgs.name = "unknown"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ // list all instances of an alternate type and expect failure
+ registryArgs.list = true;
+ registryArgs.name = null
+ registryArgs.serviceType = "org-apache-hadoop"
+ try {
+ client.actionRegistryListYarn(registryArgs)
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+
+ registryArgs.serviceType = ""
+
+ //set the name
+ registryArgs.name = clustername;
+ registryArgs.serviceType = SliderKeys.APP_TYPE
+
+
+ //now expect list to work
+ describe registryArgs.toString()
+
+ def listedInstance = client.actionRegistryListYarn(registryArgs)
+ assert listedInstance[0].id == serviceInstance.id
+
+
+ // listconf
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+ // listconf --internal
+ registryArgs.list = false;
+ registryArgs.listConf = true
+ registryArgs.internal = true
+ describe registryArgs.toString()
+ assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ registryArgs.list = false;
+ registryArgs.listConf = false
+ registryArgs.internal = false
+
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registryArgs.getConf = yarn_site_config
+
+ //properties format
+ registryArgs.format = "properties"
+ describe registryArgs.toString()
+
+ client.actionRegistry(registryArgs)
+
+
+ File outputDir = new File("target/test_standalone_registry_am/output")
+ outputDir.mkdirs()
+
+ // create a new registry args with the defaults back in
+ registryArgs = new ActionRegistryArgs(clustername)
+ registryArgs.getConf = yarn_site_config
+ registryArgs.dest = outputDir
+ describe registryArgs.toString()
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".xml").exists()
+
+ registryArgs.format = "properties"
+ client.actionRegistry(registryArgs)
+ assert new File(outputDir, yarn_site_config + ".properties").exists()
+
+ describe registryArgs.toString()
+
+ def unknownFilename = "undefined-file"
+ registryArgs.getConf = unknownFilename
+ assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
+
+ describe "stop cluster"
+ //now kill that cluster
+ assert 0 == clusterActionFreeze(client, clustername)
+ //list it & See if it is still there
+ ApplicationReport oldInstance = yarnRegistryClient.findInstance(
+ clustername)
+ assert oldInstance != null
+ assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
+
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5784c4ab/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
deleted file mode 100644
index bd15bb4..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestYarnRegistryAM.groovy
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.PathNotFoundException
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.hadoop.yarn.registry.client.api.RegistryConstants
-import org.apache.hadoop.yarn.registry.client.binding.RecordOperations
-import org.apache.hadoop.yarn.registry.client.binding.RegistryTypeUtils
-import org.apache.hadoop.yarn.registry.client.types.RegistryPathStatus
-
-import static org.apache.hadoop.yarn.registry.client.binding.BindingUtils.*
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.api.ClusterNode
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.params.ActionRegistryArgs
-import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.core.persist.JsonSerDeser
-import org.apache.slider.core.registry.docstore.PublishedConfigSet
-import org.apache.slider.core.registry.docstore.PublishedConfiguration
-import org.apache.slider.core.registry.docstore.UriMap
-import org.apache.slider.core.registry.info.CustomRegistryConstants
-import org.apache.slider.core.registry.retrieve.RegistryRetriever
-import org.apache.slider.server.appmaster.PublishedArtifacts
-import org.apache.slider.server.appmaster.web.rest.RestPaths
-import org.junit.Test
-
-/**
- * work with a YARN registry
- */
-@CompileStatic
-@Slf4j
-
-class TestYarnRegistryAM extends AgentMiniClusterTestBase {
-
-
- public static final String ARTIFACT_NAME = PublishedArtifacts.COMPLETE_CONFIG
-
- @Test
- public void testYarnRegistryAM() throws Throwable {
-
-
- describe "create a masterless AM then perform YARN registry operations on it"
-
-
- String clustername = createMiniCluster(configuration, 1, true)
-
- // get local binding
- def registryOperations = microZKCluster.registryOperations
- registryOperations.stat(RegistryConstants.PATH_SYSTEM_SERVICES)
-
- // verify the cluster has the YARN reg service live
- def rmRegistryService = miniCluster.getResourceManager(0).getRMContext().registry
- assert rmRegistryService
-
-
-
-
-
- ServiceLauncher<SliderClient> launcher
- launcher = createStandaloneAM(clustername, true, false)
- SliderClient client = launcher.service
- addToTeardown(client);
-
- ApplicationReport report = waitForClusterLive(client)
- logReport(report)
- List<ApplicationReport> apps = client.applications;
-
- List<ClusterNode> clusterNodes = client.listClusterNodesInRole(
- SliderKeys.COMPONENT_AM)
- assert ((List<ClusterNode>)clusterNodes).size() == 1
-
- ClusterNode masterNode = clusterNodes[0]
- log.info("Master node = ${masterNode}");
-
- List<ClusterNode> nodes
- String[] uuids = client.listNodeUUIDsByRole(SliderKeys.COMPONENT_AM)
- assert uuids.length == 1;
- nodes = client.listClusterNodes(uuids);
- assert ((List<ClusterNode>)nodes).size() == 1;
- describe "AM Node UUID=${uuids[0]}"
-
- nodes = listNodesInRole(client, SliderKeys.COMPONENT_AM)
- assert ((List<ClusterNode>)nodes).size() == 1;
- nodes = listNodesInRole(client, "")
- assert ((List<ClusterNode>)nodes).size() == 1;
- ClusterNode master = nodes[0]
- assert master.role == SliderKeys.COMPONENT_AM
-
-
-
-
- String username = client.username
- def yarnRegistryClient = client.yarnAppListClient
- describe("list of all applications")
- logApplications(apps)
- describe("apps of user $username")
- List<ApplicationReport> userInstances = yarnRegistryClient.listInstances()
- logApplications(userInstances)
- assert userInstances.size() == 1
- describe("named app $clustername")
- ApplicationReport instance = yarnRegistryClient.findInstance(clustername)
- logReport(instance)
- assert instance != null
-
- // sleep to allow registration to complete
- sleep(5000)
-
-
-
-
- try {
- def yarnRegistryDump = client.dumpYarnRegistry(true).toString()
- log.info("yarn service registry: \n${yarnRegistryDump}\n")
- } catch (IOException ignored) {
-
- }
-
-
- describe "service registry names"
- def registryService = client.registryOperations
-
- def self = currentUser()
- RegistryPathStatus[] serviceTypes = registryService.listDir(userPath(self))
- dumpArray(serviceTypes)
-
- def recordsPath = serviceclassPath(self, SliderKeys.APP_TYPE)
-
- def serviceRecords = RecordOperations.extractServiceRecords(registryService,
- registryService.listDir(recordsPath))
- dumpCollection(serviceRecords)
- assert serviceRecords.size() == 1
-
- def serviceInstance = serviceRecords[0]
- log.info(serviceInstance.toString())
-
- assert 2 <= serviceInstance.external.size()
-
- // hit the registry web page
-
- def registryEndpoint = serviceInstance.getExternalEndpoint(
- CustomRegistryConstants.REGISTRY_REST_API)
- assert registryEndpoint != null
- def registryURL = RegistryTypeUtils.retrieveAddressURLs(registryEndpoint)[0]
- describe("Registry WADL @ $registryURL")
-
- def publisherEndpoint = serviceInstance.getExternalEndpoint(
- CustomRegistryConstants.PUBLISHER_REST_API)
-
- def publisherURL = RegistryTypeUtils.retrieveAddressURLs(publisherEndpoint)[0]
- def publisher = publisherURL.toString()
- describe("Publisher")
-
- JsonSerDeser<UriMap> uriMapDeser = new JsonSerDeser<>(UriMap)
- def setlisting = GET(publisherURL)
-
- log.info(setlisting)
-
- UriMap uris = uriMapDeser.fromJson(setlisting)
- assert uris.uris[RestPaths.SLIDER_CONFIGSET]
- def publishedJSON = GET(publisherURL, RestPaths.SLIDER_CONFIGSET)
- JsonSerDeser< PublishedConfigSet> serDeser= new JsonSerDeser<>(
- PublishedConfigSet)
- def configSet = serDeser.fromJson(publishedJSON)
- assert configSet.size() >= 1
- assert configSet.contains(ARTIFACT_NAME)
- PublishedConfiguration publishedYarnSite = configSet.get(ARTIFACT_NAME)
-
- assert publishedYarnSite.empty
-
- //get the full URL
- def yarnSitePublisher = appendToURL(publisher,
- RestPaths.SLIDER_CONFIGSET,
- ARTIFACT_NAME)
-
- String confJSON = GET(yarnSitePublisher)
-// log.info(confJSON)
- JsonSerDeser< PublishedConfiguration> confSerDeser =
- new JsonSerDeser<PublishedConfiguration>(PublishedConfiguration)
-
- publishedYarnSite = confSerDeser.fromJson(confJSON)
-
- assert !publishedYarnSite.empty
-
-
- //get the XML
- def yarnSiteXML = yarnSitePublisher + ".xml"
-
-
- String confXML = GET(yarnSiteXML)
- log.info("Conf XML at $yarnSiteXML = \n $confXML")
-
- String properties = GET(yarnSitePublisher + ".properties")
- Properties parsedProps = new Properties()
- parsedProps.load(new StringReader(properties))
- assert parsedProps.size() > 0
- def rmAddrFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_ADDRESS)
- def rmHostnameFromDownloadedProperties = parsedProps.get(YarnConfiguration.RM_HOSTNAME)
- assert rmAddrFromDownloadedProperties
- assert rmHostnameFromDownloadedProperties
-
- String json = GET(yarnSitePublisher + ".json")
-
-
-
- describe("Registry List")
- log.info(GET(registryURL))
-
-
- describe "Registry Retrieval Class"
- // retrieval
-
- RegistryRetriever retriever = new RegistryRetriever(serviceInstance)
- log.info retriever.toString()
-
- assert retriever.hasConfigurations(true)
- PublishedConfigSet externalConfSet = retriever.getConfigurations(true)
- dumpConfigurationSet(externalConfSet)
- assert externalConfSet[ARTIFACT_NAME]
-
-
- describe "verify SLIDER-52 processing"
- def yarnSite = retriever.retrieveConfiguration(
- externalConfSet,
- ARTIFACT_NAME,
- true)
- assert !yarnSite.empty
- def siteXML = yarnSite.asConfiguration()
- def rmHostnameViaClientSideXML = parsedProps.get(
- YarnConfiguration.RM_HOSTNAME)
- assert rmHostnameViaClientSideXML == rmHostnameFromDownloadedProperties
- def rmAddrViaClientSideXML = siteXML.get(YarnConfiguration.RM_ADDRESS)
-
- log.info("RM from downloaded props = $rmAddrFromDownloadedProperties")
- assert rmAddrViaClientSideXML == rmAddrFromDownloadedProperties
-
- describe "fetch missing artifact"
- try {
- retriever.retrieveConfiguration(externalConfSet, "no-such-artifact", true)
- fail("expected a failure")
- } catch (FileNotFoundException expected) {
- // expected
- }
- describe "Internal configurations"
- assert !retriever.hasConfigurations(false)
- try {
- retriever.getConfigurations(false)
- fail("expected a failure")
- } catch (FileNotFoundException expected) {
- // expected
- }
-
-
- // retrieval via API
- ActionRegistryArgs registryArgs = new ActionRegistryArgs()
- registryArgs.verbose = true
-
- // list all
- registryArgs.list = true;
- describe registryArgs.toString()
- client.actionRegistry(registryArgs)
-
- // list a named instance and expect a failure
- registryArgs.list = true;
- registryArgs.name = "unknown"
- try {
- client.actionRegistryListYarn(registryArgs)
- } catch (PathNotFoundException expected) {
- // expected
- }
-
- // list all instances of an alternate type and expect failure
- registryArgs.list = true;
- registryArgs.name = null
- registryArgs.serviceType = "org-apache-hadoop"
- try {
- client.actionRegistryListYarn(registryArgs)
- } catch (PathNotFoundException expected) {
- // expected
- }
-
- registryArgs.serviceType = ""
-
- //set the name
- registryArgs.name = clustername;
- registryArgs.serviceType = SliderKeys.APP_TYPE
-
-
- //now expect list to work
- describe registryArgs.toString()
-
- def listedInstance = client.actionRegistryListYarn(registryArgs)
- assert listedInstance[0].id == serviceInstance.id
-
-
- // listconf
- registryArgs.list = false;
- registryArgs.listConf = true
- describe registryArgs.toString()
-
- client.actionRegistry(registryArgs)
-
- // listconf --internal
- registryArgs.list = false;
- registryArgs.listConf = true
- registryArgs.internal = true
- describe registryArgs.toString()
- assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
-
- registryArgs.list = false;
- registryArgs.listConf = false
- registryArgs.internal = false
-
- def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
- registryArgs.getConf = yarn_site_config
-
- //properties format
- registryArgs.format = "properties"
- describe registryArgs.toString()
-
- client.actionRegistry(registryArgs)
-
-
- File outputDir = new File("target/test_standalone_registry_am/output")
- outputDir.mkdirs()
-
- // create a new registry args with the defaults back in
- registryArgs = new ActionRegistryArgs(clustername)
- registryArgs.getConf = yarn_site_config
- registryArgs.dest = outputDir
- describe registryArgs.toString()
- client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".xml").exists()
-
- registryArgs.format = "properties"
- client.actionRegistry(registryArgs)
- assert new File(outputDir, yarn_site_config + ".properties").exists()
-
- describe registryArgs.toString()
-
- def unknownFilename = "undefined-file"
- registryArgs.getConf = unknownFilename
- assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
-
- describe "stop cluster"
- //now kill that cluster
- assert 0 == clusterActionFreeze(client, clustername)
- //list it & See if it is still there
- ApplicationReport oldInstance = yarnRegistryClient.findInstance(
- clustername)
- assert oldInstance != null
- assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
-
-
-
- }
-}
[17/47] git commit: SLIDER-350 windows support
Posted by st...@apache.org.
SLIDER-350 windows support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/93c3c770
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/93c3c770
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/93c3c770
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 93c3c770fed878895e4fb67ac17cbbb098cb4fb3
Parents: 8f9caa9
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 14:05:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 14:05:38 2014 +0100
----------------------------------------------------------------------
.../common/tools/TestWindowsSupport.groovy | 22 +++++++++++++++-----
1 file changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/93c3c770/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 61ede62..6ae683a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -109,13 +109,13 @@ class TestWindowsSupport extends SliderTestBase {
@Test
public void testHasGawkInstalled() throws Throwable {
assume(Shell.WINDOWS, "not windows")
- assert 0 == exec(["gawk", "--version"])
+ exec(0, ["gawk", "--version"])
}
@Test
public void testHasXargsInstalled() throws Throwable {
assume(Shell.WINDOWS, "not windows")
- assert 0 == exec(["xargs", "--version"])
+ exec(0, ["xargs", "--version"])
}
@@ -145,16 +145,28 @@ class TestWindowsSupport extends SliderTestBase {
File winUtils = new File(winUtilsPath)
log.debug("Winutils is at $winUtils)")
- ForkedProcessService process = exec([winUtilsPath, "systeminfo"])
- assert 0 == process.exitCode
+ exec(0, [winUtilsPath, "systeminfo"])
}
/**
* Exec a set of commands, wait a few seconds for it to finish.
+ * @param status code
* @param commands
- * @return
+ * @return the process
*/
+ public ForkedProcessService exec(int status, List<String> commands) {
+ ForkedProcessService process = exec(commands)
+ assert status == process.exitCode
+ return process
+ }
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+
public ForkedProcessService exec(List<String> commands) {
ForkedProcessService process;
process = new ForkedProcessService(
[32/47] git commit: SLIDER-395 testZKIntegration broken
Posted by st...@apache.org.
SLIDER-395 testZKIntegration broken
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b7f37ea2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b7f37ea2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b7f37ea2
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b7f37ea2b1f6842fab57b37a6b54b08209b895aa
Parents: bd0cdd1
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 19:00:42 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 19:00:42 2014 +0100
----------------------------------------------------------------------
.../slider/common/tools/TestZKIntegration.groovy | 16 +++++++++++-----
.../slider/test/YarnZKMiniClusterTestBase.groovy | 2 +-
2 files changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7f37ea2/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index c13af78..836edbd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -48,7 +48,8 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Test
public void testIntegrationCreate() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "cluster1", true, false, 5000)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "cluster1", true, false, 5000)
String userPath = ZKIntegration.mkSliderUserPath(USER)
Stat stat = zki.stat(userPath)
assert stat != null
@@ -59,7 +60,8 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
public void testListUserClustersWithoutAnyClusters() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "", true, false, 5000)
String userPath = ZKIntegration.mkSliderUserPath(USER)
List<String> clusters = zki.clusters
assert clusters.empty
@@ -69,7 +71,8 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
public void testListUserClustersWithOneCluster() throws Throwable {
assertHasZKCluster()
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "", true, false, 5000)
String userPath = ZKIntegration.mkSliderUserPath(USER)
String fullPath = zki.createPath(userPath, "/cluster-",
ZooDefs.Ids.OPEN_ACL_UNSAFE,
@@ -82,7 +85,8 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Test
public void testListUserClustersWithTwoCluster() throws Throwable {
- ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
+ ZKIntegration zki = createZKIntegrationInstance(
+ getZKBinding(), "", true, false, 5000)
String userPath = ZKIntegration.mkSliderUserPath(USER)
String c1 = createEphemeralChild(zki, userPath)
log.info("Ephemeral path $c1")
@@ -110,6 +114,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
path = client.createZookeeperNode("cl1", false)
zki = client.lastZKIntegration
+ assert zki
assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
assert zki.exists(zkPath)
@@ -135,7 +140,8 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Override
protected ZKIntegration getZkClient(String clusterName, String user) {
- createZKIntegrationInstance(ZKBinding, "cl1", true, false, 5000)
+ zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000)
+ return zki;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7f37ea2/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index 0259fb7..9b5f082 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -147,7 +147,7 @@ public abstract class YarnZKMiniClusterTestBase extends YarnMiniClusterTestBase
assert microZKCluster != null
}
- protected String getZKBinding() {
+ public String getZKBinding() {
if (!microZKCluster) {
return "localhost:1"
} else {
[03/47] git commit: SLIDER-388 allow default zookeeper port in
ZookeeperUtils
Posted by st...@apache.org.
SLIDER-388 allow default zookeeper port in ZookeeperUtils
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b525287d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b525287d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b525287d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b525287dd05a29af6308890aef44d524a6341296
Parents: 0ac2536
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Sep 2 12:39:06 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue Sep 2 12:39:06 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b525287d/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
index 61b1ff0..cc1b2c9 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
@@ -27,6 +27,7 @@ import java.util.ArrayList;
import java.util.List;
public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
public static String buildConnectionString(String zkHosts, int port) {
String zkPort = Integer.toString(port);
@@ -73,7 +74,7 @@ public class ZookeeperUtils {
List<HostAndPort> list = new ArrayList<HostAndPort>(len);
if (strings != null) {
for (String s : strings) {
- list.add(HostAndPort.fromString(s.trim()));
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
}
}
return list;
[08/47] git commit: SLIDER-385 move shortlife property to internal
Posted by st...@apache.org.
SLIDER-385 move shortlife property to internal
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e9a353c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e9a353c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e9a353c1
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e9a353c1ab201f1481e224b1e2b3e03188b9b2dc
Parents: b0b7bd3
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Sep 3 09:19:21 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Sep 3 09:19:21 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/api/InternalKeys.java | 11 +++++++++++
.../main/java/org/apache/slider/api/ResourceKeys.java | 13 -------------
.../apache/slider/server/appmaster/state/AppState.java | 5 +++--
3 files changed, 14 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e9a353c1/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index ad384e2..b58f1c6 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -75,6 +75,17 @@ public interface InternalKeys {
*/
int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
/**
+ * Time in seconds before a container is considered long-lived.
+ * Shortlived containers are interpreted as a problem with the role
+ * and/or the host: {@value}
+ */
+ String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
+ "internal.container.failure.shortlife";
+ /**
+ * Default short life threshold: {@value}
+ */
+ int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
+ /**
* Version of the app: {@value}
*/
String KEYTAB_LOCATION = "internal.keytab.location";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e9a353c1/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index 3d54140..b542f1a 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -92,19 +92,6 @@ public interface ResourceKeys {
/**
- * Time in seconds before a container is considered long-lived.
- * Shortlived containers are interpreted as a problem with the role
- * and/or the host: {@value}
- */
- String CONTAINER_FAILURE_SHORTLIFE =
- "container.failure.shortlife";
-
- /**
- * Default short life threshold: {@value}
- */
- int DEFAULT_CONTAINER_FAILURE_SHORTLIFE = 60;
-
- /**
* maximum number of failed containers (in a single role)
* before the cluster is deemed to have failed {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e9a353c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index dce48d1..3e6bb74 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -36,6 +36,7 @@ import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
@@ -520,8 +521,8 @@ public class AppState {
instanceDefinition.getResourceOperations().getGlobalOptions();
startTimeThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+ InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
[26/47] git commit: Merge branch 'feature/SLIDER-394_env_check' into
develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-394_env_check' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/88d72ddf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/88d72ddf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/88d72ddf
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 88d72ddfb32977ff098b0329ae6f3faf2f7c2875
Parents: 6320f61 2d6f276
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 17:12:02 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 17:12:02 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 18 +-
.../common/params/AbstractActionArgs.java | 11 +
.../slider/common/params/ActionHelpArgs.java | 14 +-
.../slider/common/params/ActionVersionArgs.java | 12 +-
.../apache/slider/common/tools/SliderUtils.java | 557 +++++++++++++------
.../slider/core/build/InstanceBuilder.java | 5 +-
.../server/appmaster/SliderAppMaster.java | 5 +-
.../services/utility/EndOfServiceWaiter.java | 84 +++
.../services/workflow/ForkedProcessService.java | 21 +-
.../services/workflow/LongLivedProcess.java | 57 +-
.../agent/AgentMiniClusterTestBase.groovy | 9 +
.../tools/TestExecutionEnvironment.groovy | 42 ++
.../common/tools/TestWindowsSupport.groovy | 4 +-
.../slider/providers/agent/AgentTestBase.groovy | 10 +
.../services/workflow/EndOfServiceWaiter.java | 56 --
.../TestWorkflowForkedProcessService.java | 4 +-
.../masterless/TestRoleOptPropagation.groovy | 4 +-
17 files changed, 663 insertions(+), 250 deletions(-)
----------------------------------------------------------------------
[44/47] git commit: SLIDER-401 agent tests on windows
Posted by st...@apache.org.
SLIDER-401 agent tests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b323df81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b323df81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b323df81
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b323df811a138a0880b60be95d9150419d1dcb85
Parents: ce3f700
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:53:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:53:05 2014 +0100
----------------------------------------------------------------------
slider-agent/src/test/python/unitTests.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b323df81/slider-agent/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
index bab121f..6aa0167 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -70,8 +70,9 @@ def all_tests_suite():
else:
for file_name in files_list:
if file_name.endswith(PY_EXT) and not file_name == __file__:
- logger.info(file_name)
- tests_list.append(file_name.replace(PY_EXT, ''))
+ replaced = file_name.replace(PY_EXT, '')
+ logger.info(replaced)
+ tests_list.append(replaced)
logger.info('------------------------------------------------------------------------')
suite = unittest.TestLoader().loadTestsFromNames(tests_list)
[10/47] git commit: SLIDER-181. Storm package updates
Posted by st...@apache.org.
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/38c816eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/38c816eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/38c816eb
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 38c816ebbd3c883308bbe36fc00846d4ff4dac7b
Parents: e9a353c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:15:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:46:02 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 19 ++---
app-packages/storm/appConfig.json | 10 +--
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ----
app-packages/storm/pom.xml | 90 ++++++++++++++++++++
app-packages/storm/src/assembly/storm.xml | 68 +++++++++++++++
6 files changed, 172 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index d4df85f..49adfce 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,19 +17,16 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/)
- zip -r apache-storm-0.9.3.zip .
-
-Verify the content using
- unzip -l "$@" apache-storm-0.9.3.zip
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b5fe436..df28e8f 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
- "application.def": "apache-storm-0.9.3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
@@ -22,8 +22,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +31,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 43535f4..bfd84e8 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name>
+ <name>files/${pkg.name}</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..c08ed3d
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/38c816eb/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..47d223a
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
[25/47] git commit: SLIDER-377 AM-spawning tests do prelaunch
validity checks
Posted by st...@apache.org.
SLIDER-377 AM-spawning tests do prelaunch validity checks
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2d6f276d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2d6f276d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2d6f276d
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2d6f276d16cc93648ccf7306eb94f5ac09bb5763
Parents: 0164277
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 17:09:12 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 17:09:12 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/agent/AgentMiniClusterTestBase.groovy | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2d6f276d/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 4566d9d..7786c41 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -27,6 +27,7 @@ import org.apache.commons.io.FileUtils
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
import org.apache.slider.test.YarnZKMiniClusterTestBase
@@ -47,6 +48,14 @@ extends YarnZKMiniClusterTestBase {
protected static Map<String, String> agentDefOptions
private static TemporaryFolder tempFolder = new TemporaryFolder();
+ /**
+ * Server side test: validate system env before launch
+ */
+ @BeforeClass
+ public static void checkSystem() {
+ SliderUtils.validateSliderServerEnvironment(null)
+ }
+
@BeforeClass
public static void createSubConfFiles() {
[35/47] git commit: SLIDER-397 taskkill /f
Posted by st...@apache.org.
SLIDER-397 taskkill /f
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/27f0a625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/27f0a625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/27f0a625
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 27f0a625265de8e0d818917c94b2f0d5385b9044
Parents: 132d9f2
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 20:39:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 20:39:05 2014 +0100
----------------------------------------------------------------------
.../test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/27f0a625/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 89f1a34..5086248 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -747,7 +747,7 @@ class SliderTestUtils extends Assert {
def commandString
if (!Shell.WINDOWS) {
- GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill $signal"
+ GString killCommand = "jps -l| grep ${grepString} | awk '{print \$1}' | xargs kill /f $signal"
log.info("Command command = $killCommand" )
commandString = ["bash", "-c", killCommand]
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /PID\""
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID "
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[24/47] git commit: SLIDER-377 AM checks for validity of server side;
client does this if the CLI action says it should (default is yes)
Posted by st...@apache.org.
SLIDER-377 AM checks for validity of server side; client does this if the CLI action says it should (default is yes)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/01642770
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/01642770
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/01642770
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0164277025ce3f8ec0a53b4dbd1005967632262e
Parents: b636180
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 17:04:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 17:04:46 2014 +0100
----------------------------------------------------------------------
.../apache/slider/providers/agent/AgentTestBase.groovy | 10 ++++++++++
.../minicluster/masterless/TestRoleOptPropagation.groovy | 4 +---
2 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01642770/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 6dee64f..2953381 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -26,9 +26,11 @@ import org.apache.commons.compress.utils.IOUtils
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Before
+import org.junit.BeforeClass
import org.junit.Rule
import org.junit.rules.TemporaryFolder
@@ -47,6 +49,14 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ /**
+ * Server side test: validate system env before launch
+ */
+ @BeforeClass
+ public static void checkSystem() {
+ SliderUtils.validateSliderServerEnvironment(null)
+ }
+
public String app_def_pkg_path;
@Before
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01642770/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
index cffde39..dd9b8e9 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
@@ -30,8 +30,6 @@ import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
-import static HBaseKeys.PROVIDER_HBASE
-import static Arguments.ARG_PROVIDER
@CompileStatic
@Slf4j
@@ -54,7 +52,7 @@ class TestRoleOptPropagation extends HBaseMiniClusterTestBase {
],
[
Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_MASTER, ENV, "4",
- ARG_PROVIDER, PROVIDER_HBASE
+ Arguments.ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE
],
true,
true,
[39/47] git commit: SLIDER-399. Agent command doesn't consider the
case where LOG_DIRS contains multiple directories
Posted by st...@apache.org.
SLIDER-399. Agent command doesn't consider the case where LOG_DIRS contains multiple directories
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4c808204
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4c808204
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4c808204
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 4c808204d749f1504b768b12a98bf64f583e49d3
Parents: 02faa94
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 4 21:50:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 4 21:50:52 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/providers/agent/AgentKeys.java | 1 +
.../org/apache/slider/providers/agent/AgentProviderService.java | 3 +--
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c808204/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 1622367..1da99b0 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -91,6 +91,7 @@ public interface AgentKeys {
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
+ String AGENT_OUT_FILE = "agent.out";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4c808204/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d188d49..02ef625 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -333,8 +333,7 @@ public class AgentProviderService extends AbstractProviderService implements
operation.add(debugCmd);
}
- String outfile = new File(logDir, "agent.out").toString();
- operation.add("> " + outfile + " 2>&1");
+ operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + AgentKeys.AGENT_OUT_FILE + " 2>&1");
launcher.addCommand(operation.build());
[38/47] git commit: SLIDER-400 added zero port properties to sample
appConfigs
Posted by st...@apache.org.
SLIDER-400 added zero port properties to sample appConfigs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/168a7af0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/168a7af0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/168a7af0
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 168a7af0251abb7d4b1c8017360fc21531ca83b7
Parents: e2a8032
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu Sep 4 20:22:40 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Sep 4 20:22:40 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 2 ++
app-packages/accumulo/src/test/resources/appConfig_kerberos.json | 2 ++
.../accumulo/src/test/resources/appConfig_monitor_ssl.json | 2 ++
app-packages/accumulo/src/test/resources/appConfig_ssl.json | 2 ++
4 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index 2c0c2d5..8828a64 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -40,6 +40,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
index c9684d4..b1bff37 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
@@ -42,6 +42,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
index 45e6e19..61a9bb1 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_monitor_ssl.json
@@ -40,6 +40,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/168a7af0/app-packages/accumulo/src/test/resources/appConfig_ssl.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_ssl.json b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
index 3daaaaf..e46d40e 100644
--- a/app-packages/accumulo/src/test/resources/appConfig_ssl.json
+++ b/app-packages/accumulo/src/test/resources/appConfig_ssl.json
@@ -42,6 +42,8 @@
"site.accumulo-site.gc.port.client": "0",
"site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
"site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
"site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
},
"credentials": {
[12/47] git commit: SLIDER-391. appConfig.json should allow reference
to other config properties and resolve them automatically
Posted by st...@apache.org.
SLIDER-391. appConfig.json should allow reference to other config properties and resolve them automatically
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/02faa944
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/02faa944
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/02faa944
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 02faa944da56cd6252b48afbdb6ee427291f1fe0
Parents: 26b4153
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 18:17:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 18:17:13 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 8 +++--
app-packages/storm/appConfig.json | 7 ++--
.../providers/agent/AgentProviderService.java | 35 ++++++++++++++++----
.../agent/TestAgentProviderService.java | 29 ++++++++++++++++
4 files changed, 66 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 7df0dc3..07d1b4e 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -11,14 +11,16 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
+
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.monitor_protocol": "http",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index df28e8f..ac9bd49 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -11,6 +11,7 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
+
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
@@ -22,8 +23,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${site.global.ganglia_server_host},port=${site.global.ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 61866fb..d188d49 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1196,7 +1196,8 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd, containerId);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
+ cmd.setConfigurations(configurations);
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
@@ -1246,12 +1247,6 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
- cmd.setConfigurations(configurations);
- }
-
@VisibleForTesting
protected void addStatusCommand(String roleName,
String containerId,
@@ -1380,9 +1375,35 @@ public class AgentProviderService extends AbstractProviderService implements
configurations, tokens, containerId);
}
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
return configurations;
}
+ protected void dereferenceAllConfigs(Map<String, Map<String, String>> configurations) {
+ Map<String, String> allConfigs = new HashMap<String, String>();
+ String lookupFormat = "${site.%s.%s}";
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ allConfigs.put(String.format(lookupFormat, configType, configName), configBucket.get(configName));
+ }
+ }
+
+ for (String configType : configurations.keySet()) {
+ Map<String, String> configBucket = configurations.get(configType);
+ for (String configName : configBucket.keySet()) {
+ String configValue = configBucket.get(configName);
+ for (String lookUpKey : allConfigs.keySet()) {
+ if (configValue != null && configValue.contains(lookUpKey)) {
+ configBucket.put(configName, configValue.replace(lookUpKey, allConfigs.get(lookUpKey)));
+ }
+ }
+ }
+ }
+ }
+
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/02faa944/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 16f5347..0f0e744 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -1128,4 +1128,33 @@ public class TestAgentProviderService {
Assert.assertFalse(configs.contains("bore-site"));
}
+ @Test
+ public void testDereferenceAllConfig() {
+ AgentProviderService aps = new AgentProviderService();
+ Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>();
+ Map<String, String> cfg1 = new HashMap<String, String>();
+ cfg1.put("a1", "${site.cfg-2.A1}");
+ cfg1.put("b1", "22");
+ cfg1.put("c1", "33");
+ cfg1.put("d1", "${site.cfg1.c1}AA");
+ Map<String, String> cfg2 = new HashMap<String, String>();
+ cfg2.put("A1", "11");
+ cfg2.put("B1", "${site.cfg-2.A1},${site.cfg-2.A1}");
+ cfg2.put("C1", "DD${site.cfg1.c1}");
+ cfg2.put("D1", "${14}");
+
+ allConfigs.put("cfg1", cfg1);
+ allConfigs.put("cfg-2", cfg2);
+ aps.dereferenceAllConfigs(allConfigs);
+ Assert.assertEquals("11", cfg1.get("a1"));
+ Assert.assertEquals("22", cfg1.get("b1"));
+ Assert.assertEquals("33", cfg1.get("c1"));
+ Assert.assertEquals("33AA", cfg1.get("d1"));
+
+ Assert.assertEquals("11", cfg2.get("A1"));
+ Assert.assertEquals("11,11", cfg2.get("B1"));
+ Assert.assertEquals("DD33", cfg2.get("C1"));
+ Assert.assertEquals("${14}", cfg2.get("D1"));
+ }
+
}
[23/47] git commit: SLIDER-377 AM checks for validity of server side;
client does this if the CLI action says it should (default is yes)
Posted by st...@apache.org.
SLIDER-377 AM checks for validity of server side; client does this if the CLI action says it should (default is yes)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b6361803
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b6361803
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b6361803
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b636180392a719b1b663f77592c23b7b16a8f4d3
Parents: 8034012
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 16:53:35 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 16:53:35 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 18 ++++++++++++++----
.../slider/common/params/AbstractActionArgs.java | 11 +++++++++++
.../slider/common/params/ActionHelpArgs.java | 14 ++++++++++++--
.../slider/common/params/ActionVersionArgs.java | 12 +++++++++++-
.../apache/slider/common/tools/SliderUtils.java | 2 +-
.../apache/slider/core/build/InstanceBuilder.java | 5 +++--
.../slider/server/appmaster/SliderAppMaster.java | 2 +-
.../services/workflow/LongLivedProcess.java | 3 ++-
8 files changed, 55 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f6d761e..f4040c4 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -46,6 +46,7 @@ import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
@@ -291,6 +292,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// choose the action
String action = serviceArgs.getAction();
+
+ AbstractActionArgs coreAction = serviceArgs.getCoreAction();
+ if (coreAction.getHadoopServicesRequired()) {
+ // validate the client
+ SliderUtils.validateSliderClientEnvironment(null);
+ }
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -613,7 +620,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// verify that a live cluster isn't there
SliderUtils.validateClusterName(clustername);
verifyBindingsDefined();
- if (!liveClusterAllowed) verifyNoLiveClusters(clustername);
+ if (!liveClusterAllowed) {
+ verifyNoLiveClusters(clustername);
+ }
Configuration conf = getConfig();
String registryQuorum = lookupZKQuorum();
@@ -1169,7 +1178,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
addConfOptionToCLI(commandLine,
config,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -1263,10 +1272,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
private void propagatePrincipals(Configuration config,
AggregateConf clusterSpec) {
- String dfsPrincipal = config.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ String dfsPrincipal = config.get(
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index f4a4569..d18c85b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -148,4 +148,15 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
public String toString() {
return super.toString() + ": " + getActionName();
}
+
+ /**
+ * Override point:
+ * Flag to indicate that core hadoop API services are needed (HDFS, YARN, etc)
+ * —and that validation of the client state should take place.
+ *
+ * @return a flag to indicate that the core hadoop services will be needed.
+ */
+ public boolean getHadoopServicesRequired() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
index 4d4098f..f59396a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionHelpArgs.java
@@ -20,9 +20,11 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
+/**
+ * The Help command
+ */
@Parameters(commandNames = {SliderActions.ACTION_HELP, SliderActions.ACTION_USAGE},
- commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
+ commandDescription = SliderActions.DESCRIBE_ACTION_HELP)
public class ActionHelpArgs extends AbstractActionArgs {
@Override
public String getActionName() {
@@ -38,4 +40,12 @@ public class ActionHelpArgs extends AbstractActionArgs {
return 0;
}
+ /**
+ * This action does not need hadoop services
+ * @return false
+ */
+ @Override
+ public boolean getHadoopServicesRequired() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
index c36ef62..b9d212b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionVersionArgs.java
@@ -20,9 +20,11 @@ package org.apache.slider.common.params;
import com.beust.jcommander.Parameters;
+/**
+ * The version command
+ */
@Parameters(commandNames = {SliderActions.ACTION_VERSION},
commandDescription = SliderActions.DESCRIBE_ACTION_VERSION)
-
public class ActionVersionArgs extends AbstractActionArgs {
@Override
public String getActionName() {
@@ -33,4 +35,12 @@ public class ActionVersionArgs extends AbstractActionArgs {
return 0;
}
+ /**
+ * This action does not need hadoop services
+ * @return false
+ */
+ @Override
+ public boolean getHadoopServicesRequired() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 44fe4fd..4cdc580 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1026,7 +1026,7 @@ public final class SliderUtils {
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 937b777..bfa7097 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -181,10 +181,11 @@ public class InstanceBuilder {
* Propagate any critical principals from the current site config down to the HBase one.
*/
public void propagatePrincipals() {
- String dfsPrincipal = conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ String dfsPrincipal = conf.get(
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY;
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 49c7792..1c60b69 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -390,7 +390,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
log.debug("Authenticating as {}", ugi);
SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY);
+ DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
// always enforce protocol to be token-based.
conf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6361803/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index a0df2c9..57b989c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -284,7 +284,8 @@ public class LongLivedProcess implements Runnable {
/**
* Exec the process
* @return the process
- * @throws IOException
+ * @throws IOException on aany failure to start the process
+ * @throws FileNotFoundException if the process could not be found
*/
private Process spawnChildProcess() throws IOException {
if (process != null) {
[45/47] git commit: Merge branch
'feature/SLIDER-401_slider-agent_tests_on_windows' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-401_slider-agent_tests_on_windows' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f14d32c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f14d32c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f14d32c6
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: f14d32c6fac2e252e587dbdc88e0e3e079f44d37
Parents: 2d6331a b323df8
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:59:58 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:59:58 2014 +0100
----------------------------------------------------------------------
pom.xml | 1 +
slider-agent/pom.xml | 4 +--
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/python-wrap | 40 ----------------------
slider-agent/src/test/python/python-wrap.sh | 42 ++++++++++++++++++++++++
slider-agent/src/test/python/unitTests.py | 36 ++++++++++----------
slider-core/pom.xml | 4 +--
7 files changed, 65 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
[33/47] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by st...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/930920d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/930920d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/930920d2
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 930920d245bb354cbfbec43391a203bd6db04fa4
Parents: b7f37ea 6126db6
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 19:00:55 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 19:00:55 2014 +0100
----------------------------------------------------------------------
pom.xml | 41 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[09/47] git commit: SLIDER-181. Storm package updates
Posted by st...@apache.org.
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e1129b06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e1129b06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e1129b06
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e1129b0609621356b47324dcbcabd612fd275be5
Parents: 201686e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:15:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:15:20 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 19 ++---
app-packages/storm/appConfig.json | 10 +--
app-packages/storm/metainfo.xml | 2 +-
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ----
app-packages/storm/pom.xml | 90 ++++++++++++++++++++
app-packages/storm/src/assembly/storm.xml | 68 +++++++++++++++
6 files changed, 172 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index d4df85f..49adfce 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,19 +17,16 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz.
-So if you use a different version you may need to edit a few config values.
+To create the app package you will need the Storm tarball and invoke mvn command
+with appropriate parameters.
-Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
- rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+Command:
+mvn clean package -Pstorm-app-package -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/)
- zip -r apache-storm-0.9.3.zip .
-
-Verify the content using
- unzip -l "$@" apache-storm-0.9.3.zip
+Example:
+mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
+ -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b5fe436..df28e8f 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,13 +3,13 @@
"metadata": {
},
"global": {
- "application.def": "apache-storm-0.9.3.zip",
+ "application.def": "slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
@@ -22,8 +22,8 @@
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -31,7 +31,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 43535f4..bfd84e8 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name>
+ <name>files/${pkg.name}</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
new file mode 100644
index 0000000..c08ed3d
--- /dev/null
+++ b/app-packages/storm/pom.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-storm-app-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider Storm App Package</name>
+ <description>Slider Storm App Package</description>
+ <version>${pkg.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>storm-app-package</id>
+ <build>
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/storm.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1129b06/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
new file mode 100644
index 0000000..47d223a
--- /dev/null
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>slider-storm-v${storm.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
[37/47] git commit: Merge branch
'feature/SLIDER-397_windows_test_kill()_command_failing' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-397_windows_test_kill()_command_failing' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e2a80326
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e2a80326
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e2a80326
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e2a80326a7f7ddc6f2bc8b00dbaa5265726a6bb0
Parents: 930920d ee3ad77
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 21:02:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 21:02:30 2014 +0100
----------------------------------------------------------------------
.../apache/slider/agent/standalone/TestStandaloneAMKill.groovy | 2 +-
.../test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 4 ++--
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[31/47] git commit: SLIDER-1 steve broke jenkins. reverting
Posted by st...@apache.org.
SLIDER-1 steve broke jenkins. reverting
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bd0cdd1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bd0cdd1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bd0cdd1c
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bd0cdd1c4bda4479d75f3e83f07e4802ff71e8b6
Parents: a0a3ae6
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 18:53:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 18:53:11 2014 +0100
----------------------------------------------------------------------
.../hbase/minicluster/masterless/TestRoleOptPropagation.groovy | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bd0cdd1c/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
index dd9b8e9..cffde39 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/masterless/TestRoleOptPropagation.groovy
@@ -30,6 +30,8 @@ import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
+import static HBaseKeys.PROVIDER_HBASE
+import static Arguments.ARG_PROVIDER
@CompileStatic
@Slf4j
@@ -52,7 +54,7 @@ class TestRoleOptPropagation extends HBaseMiniClusterTestBase {
],
[
Arguments.ARG_COMP_OPT, HBaseKeys.ROLE_MASTER, ENV, "4",
- Arguments.ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE
+ ARG_PROVIDER, PROVIDER_HBASE
],
true,
true,
[34/47] git commit: SLIDER-397 windows kill() command failing in
TestStandaloneAMKill
Posted by st...@apache.org.
SLIDER-397 windows kill() command failing in TestStandaloneAMKill
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/132d9f28
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/132d9f28
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/132d9f28
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 132d9f288a9d91790409bd4482731eb27cf58bba
Parents: 930920d
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 20:31:29 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 20:31:29 2014 +0100
----------------------------------------------------------------------
.../apache/slider/agent/standalone/TestStandaloneAMKill.groovy | 2 +-
.../groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/132d9f28/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
index 75f9a2c..7552394 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -58,7 +58,7 @@ class TestStandaloneAMKill extends AgentMiniClusterTestBase {
describe("listing Java processes")
lsJavaProcesses();
describe("killing AM")
- killAM(SIGTERM);
+ assert 0 == killAM(SIGTERM);
waitWhileClusterLive(sliderClient);
//give yarn some time to notice
sleep(10000)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/132d9f28/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 7ffa64a..97cc853 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -318,10 +318,10 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
/**
- * Kill all Slider Services. That i
+ * Kill all Slider Services.
* @param signal
*/
- public void killAM(int signal) {
+ public int killAM(int signal) {
killJavaProcesses(SliderAppMaster.SERVICE_CLASSNAME_SHORT, signal)
}
[43/47] git commit: SLIDER-401 agent tests on windows
Posted by st...@apache.org.
SLIDER-401 agent tests on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce3f7009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce3f7009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce3f7009
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ce3f7009e6e9fec204b264a9577a52632b9f3487
Parents: a9f0b98
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:47:50 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:47:50 2014 +0100
----------------------------------------------------------------------
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/unitTests.py | 31 ++++++++++++--------------
2 files changed, 15 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/main/python/setup.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/setup.py b/slider-agent/src/main/python/setup.py
index 421b5f9..56969b6 100644
--- a/slider-agent/src/main/python/setup.py
+++ b/slider-agent/src/main/python/setup.py
@@ -17,7 +17,7 @@ from setuptools import setup
setup(
name = "slider-agent",
- version = "0.31.0-incubating-SNAPSHOT",
+ version = "0.51.0-incubating-SNAPSHOT",
packages = ['agent'],
# metadata for upload to PyPI
author = "Apache Software Foundation",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce3f7009/slider-agent/src/test/python/unitTests.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/unitTests.py b/slider-agent/src/test/python/unitTests.py
index b01438e..bab121f 100644
--- a/slider-agent/src/test/python/unitTests.py
+++ b/slider-agent/src/test/python/unitTests.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-'''
+"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
@@ -16,14 +16,13 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-'''
+"""
import unittest
-import doctest
-from os.path import dirname, split, isdir
-import logging.handlers
+from os.path import isdir
import logging
-from random import shuffle
+import os
+import sys
LOG_FILE_NAME='tests.log'
SELECTED_PREFIX = "_"
@@ -31,9 +30,9 @@ PY_EXT='.py'
ignoredDirs = ["mock"]
class TestAgent(unittest.TestSuite):
- def run(self, result):
+ def run(self, result, debug=False):
run = unittest.TestSuite.run
- run(self, result)
+ run(self, result, debug)
return result
@@ -41,19 +40,19 @@ def parent_dir(path):
if isdir(path):
if path.endswith(os.sep):
path = os.path.dirname(path)
- parent_dir = os.path.dirname(path)
+ parent = os.path.dirname(path)
else:
- parent_dir = os.path.dirname(os.path.dirname(path))
+ parent = os.path.dirname(os.path.dirname(path))
- return parent_dir
+ return parent
def all_tests_suite():
- src_dir = os.getcwd()
+ root_dir = os.getcwd()
files_list = []
- for directory in os.listdir(src_dir):
+ for directory in os.listdir(root_dir):
if os.path.isdir(directory) and not directory in ignoredDirs:
- files_list += os.listdir(src_dir + os.sep + directory)
+ files_list += os.listdir(root_dir + os.sep + directory)
## temporarily deleting to add more predictability
## shuffle(files_list)
files_list.sort()
@@ -99,9 +98,7 @@ def main():
logger.info('------------------------------------------------------------------------')
if __name__ == '__main__':
- import os
- import sys
- import io
+
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python')
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.sep + 'main' + os.sep + 'python' + os.sep + 'agent')
[11/47] git commit: Merge branch 'develop' into
feature/auto_generate_storm_pkg
Posted by st...@apache.org.
Merge branch 'develop' into feature/auto_generate_storm_pkg
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/26b4153e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/26b4153e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/26b4153e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 26b4153ebf070521fb5030d57846f4ccd7f45cb0
Parents: e1129b0 38c816e
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 3 13:46:44 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 3 13:46:44 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/api/InternalKeys.java | 11 +++++++++++
.../main/java/org/apache/slider/api/ResourceKeys.java | 13 -------------
.../apache/slider/server/appmaster/state/AppState.java | 9 +++++----
3 files changed, 16 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[05/47] git commit: Merge branch
'feature/SLIDER-327_AM_to_move_to_optimized,
_queued_Node_review_and_refresh_action' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-327_AM_to_move_to_optimized,_queued_Node_review_and_refresh_action' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7dd217a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7dd217a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7dd217a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: e7dd217a324c205826bea4ac040b808c4ad72d61
Parents: b525287 88b665d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 2 21:02:29 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 2 21:02:29 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 9 +--
.../server/appmaster/RoleLaunchService.java | 8 +--
.../server/appmaster/SliderAppMaster.java | 66 ++++++++++++++------
.../server/appmaster/actions/ActionHalt.java | 6 +-
.../appmaster/actions/ActionKillContainer.java | 20 +++++-
.../appmaster/actions/ActionStartContainer.java | 8 ++-
.../appmaster/actions/ActionStopSlider.java | 6 +-
.../server/appmaster/actions/AsyncAction.java | 47 ++++----------
.../actions/ProviderReportedContainerLoss.java | 13 ++--
.../actions/ProviderStartupCompleted.java | 4 --
.../server/appmaster/actions/QueueAccess.java | 5 ++
.../server/appmaster/actions/QueueService.java | 11 +++-
.../actions/RegisterComponentInstance.java | 5 ++
.../actions/ReviewAndFlexApplicationSize.java | 43 +++++++++++++
.../actions/UnregisterComponentInstance.java | 7 ++-
.../server/appmaster/monkey/ChaosEntry.java | 2 +-
.../TestStandaloneAMMonkeyRestart.groovy | 8 +--
.../providers/accumulo/AccumuloTestBase.groovy | 4 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 1 -
.../flexing/TestClusterFlex1To1.groovy | 2 +-
20 files changed, 176 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
[22/47] git commit: SLIDER-394 slider client and AM services to
fail-fast if the hadoop dependencies are missing
Posted by st...@apache.org.
SLIDER-394 slider client and AM services to fail-fast if the hadoop dependencies are missing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8034012f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8034012f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8034012f
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8034012f2f0282c9dbab814a282ab47abc175204
Parents: 0f67b10
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 16:37:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 16:37:05 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/SliderAppMaster.java | 3 +++
.../server/services/workflow/LongLivedProcess.java | 16 +++++++++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8034012f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9ccb382..49c7792 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -400,6 +400,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//look at settings of Hadoop Auth, to pick up a problem seen once
checkAndWarnForAuthTokenProblems();
+
+ // validate server env
+ SliderUtils.validateSliderServerEnvironment(log);
executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
Executors.newCachedThreadPool(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8034012f/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index 6f83811..a0df2c9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -24,6 +24,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
@@ -292,7 +293,20 @@ public class LongLivedProcess implements Runnable {
if (LOG.isDebugEnabled()) {
LOG.debug("Spawning process:\n " + describeBuilder());
}
- process = processBuilder.start();
+ try {
+ process = processBuilder.start();
+ } catch (IOException e) {
+ // on windows, upconvert DOS error 2 from ::CreateProcess()
+ // to its real meaning: FileNotFound
+ if (e.toString().contains("CreateProcess error=2")) {
+ FileNotFoundException fnfe =
+ new FileNotFoundException(e.toString());
+ fnfe.initCause(e);
+ throw fnfe;
+ } else {
+ throw e;
+ }
+ }
return process;
}
[07/47] git commit: SLIDER-390 interpret short lived threshold as
seconds instead of milliseconds
Posted by st...@apache.org.
SLIDER-390 interpret short lived threshold as seconds instead of milliseconds
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b0b7bd38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b0b7bd38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b0b7bd38
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b0b7bd38542ed287415066d0215c6fda8786336e
Parents: 201686e
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Sep 3 08:38:52 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Sep 3 08:38:52 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/state/AppState.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b0b7bd38/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 8b70185..dce48d1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1227,7 +1227,7 @@ public class AppState {
/**
* Is a role short lived by the threshold set for this application
* @param instance instance
- * @return true if the instance is considered short live
+ * @return true if the instance is considered short lived
*/
@VisibleForTesting
public boolean isShortLived(RoleInstance instance) {
@@ -1236,7 +1236,7 @@ public class AppState {
boolean shortlived;
if (started > 0) {
long duration = time - started;
- shortlived = duration < startTimeThreshold;
+ shortlived = duration < (startTimeThreshold * 1000);
} else {
// never even saw a start event
shortlived = true;
[21/47] git commit: SLIDER-394 slider client and AM services to
fail-fast if the hadoop dependencies are missing
Posted by st...@apache.org.
SLIDER-394 slider client and AM services to fail-fast if the hadoop dependencies are missing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0f67b104
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0f67b104
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0f67b104
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 0f67b10424cb807aebb3d2b15c3b24149f4ae70a
Parents: 29a40f5
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 16:28:20 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 16:28:20 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 555 +++++++++++++------
.../services/utility/EndOfServiceWaiter.java | 84 +++
.../services/workflow/ForkedProcessService.java | 21 +-
.../services/workflow/LongLivedProcess.java | 38 +-
.../tools/TestExecutionEnvironment.groovy | 42 ++
.../common/tools/TestWindowsSupport.groovy | 4 +-
.../services/workflow/EndOfServiceWaiter.java | 56 --
.../TestWorkflowForkedProcessService.java | 4 +-
8 files changed, 570 insertions(+), 234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index eb214db..44fe4fd 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -56,7 +56,10 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.MissingArgException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.server.services.utility.EndOfServiceWaiter;
import org.apache.slider.server.services.utility.PatternValidator;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,6 +70,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InterruptedIOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.InetSocketAddress;
@@ -88,6 +92,7 @@ import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeSet;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -102,12 +107,24 @@ public final class SliderUtils {
* turned on (prevents re-entrancy)
*/
private static final AtomicBoolean processSecurityAlreadyInitialized =
- new AtomicBoolean(false);
+ new AtomicBoolean(false);
public static final String JAVA_SECURITY_KRB5_REALM =
"java.security.krb5.realm";
public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+
+ /**
+ * Winutils
+ */
public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
private SliderUtils() {
}
@@ -131,7 +148,8 @@ public final class SliderUtils {
* @param msg the message to be shown in exception
*/
@SuppressWarnings("ResultOfMethodCallIgnored")
- private static void validateNumber(String num, String msg) throws BadConfigException {
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
try {
Integer.parseInt(num);
} catch (NumberFormatException nfe) {
@@ -145,15 +163,16 @@ public final class SliderUtils {
* @param heapsize
* @return heapsize in MB
*/
- public static String translateTrailingHeapUnit(String heapsize) throws BadConfigException {
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
String errMsg = "Bad heapsize: ";
if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
- String num = heapsize.substring(0, heapsize.length()-1);
+ String num = heapsize.substring(0, heapsize.length() - 1);
validateNumber(num, errMsg);
return num;
}
if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
- String num = heapsize.substring(0, heapsize.length()-1)+"000";
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
validateNumber(num, errMsg);
return num;
}
@@ -222,7 +241,7 @@ public final class SliderUtils {
ClassLoader loader = my_class.getClassLoader();
if (loader == null) {
throw new IOException(
- "Class " + my_class + " does not have a classloader!");
+ "Class " + my_class + " does not have a classloader!");
}
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
Enumeration<URL> urlEnumeration = loader.getResources(class_file);
@@ -255,16 +274,16 @@ public final class SliderUtils {
}
public static void checkPort(String hostname, int port, int connectTimeout)
- throws IOException {
+ throws IOException {
InetSocketAddress addr = new InetSocketAddress(hostname, port);
checkPort(hostname, addr, connectTimeout);
}
@SuppressWarnings("SocketOpenedButNotSafelyClosed")
public static void checkPort(String name,
- InetSocketAddress address,
- int connectTimeout)
- throws IOException {
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
Socket socket = null;
try {
socket = new Socket();
@@ -274,14 +293,14 @@ public final class SliderUtils {
+ " at " + address
+ " after " + connectTimeout + "millisconds"
+ ": " + e,
- e);
+ e);
} finally {
IOUtils.closeSocket(socket);
}
}
public static void checkURL(String name, String url, int timeout) throws
- IOException {
+ IOException {
InetSocketAddress address = NetUtils.createSocketAddr(url);
checkPort(name, address, timeout);
}
@@ -294,22 +313,22 @@ public final class SliderUtils {
* @return the file
*/
public static File requiredFile(String filename, String role) throws
- IOException {
+ IOException {
if (filename.isEmpty()) {
throw new ExitUtil.ExitException(-1, role + " file not defined");
}
File file = new File(filename);
if (!file.exists()) {
throw new ExitUtil.ExitException(-1,
- role + " file not found: " +
- file.getCanonicalPath());
+ role + " file not found: " +
+ file.getCanonicalPath());
}
return file;
}
private static final PatternValidator clusternamePattern
= new PatternValidator("[a-z][a-z0-9_-]*");
-
+
/**
* Normalize a cluster name then verify that it is valid
* @param name proposed cluster name
@@ -318,12 +337,13 @@ public final class SliderUtils {
public static boolean isClusternameValid(String name) {
return name != null && clusternamePattern.matches(name);
}
+
public static boolean oldIsClusternameValid(String name) {
if (name == null || name.isEmpty()) {
return false;
}
int first = name.charAt(0);
- if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
+ if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
return false;
}
@@ -354,11 +374,11 @@ public final class SliderUtils {
* @return # of files copies
*/
public static int copyDirectory(Configuration conf,
- Path srcDirPath,
- Path destDirPath,
- FsPermission permission) throws
- IOException,
- BadClusterStateException {
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
//list all paths in the src.
@@ -366,7 +386,8 @@ public final class SliderUtils {
throw new FileNotFoundException("Source dir not found " + srcDirPath);
}
if (!srcFS.isDirectory(srcDirPath)) {
- throw new FileNotFoundException("Source dir not a directory " + srcDirPath);
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
}
GlobFilter dotFilter = new GlobFilter("[!.]*");
FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
@@ -378,7 +399,8 @@ public final class SliderUtils {
permission = FsPermission.getDirDefault();
}
if (!destFS.exists(destDirPath)) {
- new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath, permission);
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
}
Path[] sourcePaths = new Path[srcFileCount];
for (int i = 0; i < srcFileCount; i++) {
@@ -394,8 +416,8 @@ public final class SliderUtils {
sourcePaths[i] = srcFile;
}
log.debug("Copying {} files from {} to dest {}", srcFileCount,
- srcDirPath,
- destDirPath);
+ srcDirPath,
+ destDirPath);
FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
return srcFileCount;
}
@@ -430,7 +452,8 @@ public final class SliderUtils {
//if the fallback option is NOT set, enable it.
//if it is explicitly set to anything -leave alone
- if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) == null) {
+ if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) ==
+ null) {
conf.set(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, "true");
}
return conf;
@@ -469,7 +492,9 @@ public final class SliderUtils {
* @param trailing add a trailing entry or not
* @return the joined entries
*/
- public static String join(Collection collection, String separator, boolean trailing) {
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
StringBuilder b = new StringBuilder();
// fast return on empty collection
if (collection.isEmpty()) {
@@ -481,9 +506,9 @@ public final class SliderUtils {
}
int length = separator.length();
String s = b.toString();
- return (trailing || s.isEmpty())?
+ return (trailing || s.isEmpty()) ?
s
- : (b.substring(0, b.length() - length));
+ : (b.substring(0, b.length() - length));
}
/**
@@ -495,9 +520,10 @@ public final class SliderUtils {
*/
public static String join(String[] collection, String separator) {
return join(collection, separator, true);
-
-
+
+
}
+
/**
* Join an array of strings with a separator that appears after every
* instance in the list -optionally at the end
@@ -507,7 +533,7 @@ public final class SliderUtils {
* @return the joined entries
*/
public static String join(String[] collection, String separator,
- boolean trailing) {
+ boolean trailing) {
return join(Arrays.asList(collection), separator, trailing);
}
@@ -519,7 +545,7 @@ public final class SliderUtils {
* @return the list
*/
public static String joinWithInnerSeparator(String separator,
- Object... collection) {
+ Object... collection) {
StringBuilder b = new StringBuilder();
boolean first = true;
@@ -543,10 +569,15 @@ public final class SliderUtils {
return v;
}
- public static String appReportToString(ApplicationReport r, String separator) {
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
StringBuilder builder = new StringBuilder(512);
- builder.append("application ").append(
- r.getName()).append("/").append(r.getApplicationType()).append(separator);
+ builder.append("application ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
Set<String> tags = r.getApplicationTags();
if (!tags.isEmpty()) {
for (String tag : tags) {
@@ -555,12 +586,20 @@ public final class SliderUtils {
}
builder.append("state: ").append(r.getYarnApplicationState());
builder.append(separator).append("URL: ").append(r.getTrackingUrl());
- builder.append(separator).append("Started ").append(new Date(r.getStartTime()).toGMTString());
+ builder.append(separator)
+ .append("Started ")
+ .append(new Date(r.getStartTime()).toGMTString());
long finishTime = r.getFinishTime();
- if (finishTime>0) {
- builder.append(separator).append("Finished ").append(new Date(finishTime).toGMTString());
- }
- builder.append(separator).append("RPC :").append(r.getHost()).append(':').append(r.getRpcPort());
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished ")
+ .append(new Date(finishTime).toGMTString());
+ }
+ builder.append(separator)
+ .append("RPC :")
+ .append(r.getHost())
+ .append(':')
+ .append(r.getRpcPort());
String diagnostics = r.getDiagnostics();
if (!diagnostics.isEmpty()) {
builder.append(separator).append("Diagnostics :").append(diagnostics);
@@ -575,8 +614,8 @@ public final class SliderUtils {
* @param second the map that is merged in
* @return the first map
*/
- public static Map<String, String> mergeMap(Map<String, String> first,
- Map<String, String> second) {
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
first.putAll(second);
return first;
}
@@ -589,8 +628,8 @@ public final class SliderUtils {
* @return dest -with the entries merged in
*/
public static Map<String, String> mergeEntries(Map<String, String> dest,
- Iterable<Map.Entry<String, String>> entries) {
- for (Map.Entry<String, String> entry: entries) {
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
dest.put(entry.getKey(), entry.getValue());
}
return dest;
@@ -604,8 +643,8 @@ public final class SliderUtils {
* @param <T2> value type
* @return 'first' merged with the second
*/
- public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
- Map<T1, T2> second) {
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
first.putAll(second);
return first;
}
@@ -619,7 +658,7 @@ public final class SliderUtils {
* @return 'first' merged with the second
*/
public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
+ Map<T1, T2> second) {
Preconditions.checkArgument(first != null, "Null 'first' value");
Preconditions.checkArgument(second != null, "Null 'second' value");
for (Map.Entry<T1, T2> entry : second.entrySet()) {
@@ -637,8 +676,8 @@ public final class SliderUtils {
* @return a string representation of the map
*/
public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder =new StringBuilder();
- for (Map.Entry<String, String> entry: map.entrySet()) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
builder.append(entry.getKey())
.append("=\"")
.append(entry.getValue())
@@ -659,11 +698,11 @@ public final class SliderUtils {
* @throws BadConfigException if the value could not be parsed
*/
public static int getIntValue(Map<String, String> roleMap,
- String key,
- int defVal,
- int min,
- int max
- ) throws BadConfigException {
+ String key,
+ int defVal,
+ int min,
+ int max
+ ) throws BadConfigException {
String valS = roleMap.get(key);
return parseAndValidate(key, valS, defVal, min, max);
@@ -679,10 +718,10 @@ public final class SliderUtils {
* @throws BadConfigException if the value could not be parsed
*/
public static int parseAndValidate(String errorKey,
- String valS,
- int defVal,
- int min, int max) throws
- BadConfigException {
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
if (valS == null) {
valS = Integer.toString(defVal);
}
@@ -709,14 +748,14 @@ public final class SliderUtils {
public static InetSocketAddress getRmAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
}
public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
}
/**
@@ -759,11 +798,11 @@ public final class SliderUtils {
return "null container";
}
return String.format(Locale.ENGLISH,
- "ContainerID=%s nodeID=%s http=%s priority=%s",
- container.getId(),
- container.getNodeId(),
- container.getNodeHttpAddress(),
- container.getPriority());
+ "ContainerID=%s nodeID=%s http=%s priority=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority());
}
/**
@@ -844,12 +883,12 @@ public final class SliderUtils {
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
Map<String, String> env = new HashMap<String, String>();
if (roleOpts != null) {
- for (Map.Entry<String, String> entry: roleOpts.entrySet()) {
+ for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
String key = entry.getKey();
if (key.startsWith(RoleKeys.ENV_PREFIX)) {
String envName = key.substring(RoleKeys.ENV_PREFIX.length());
if (!envName.isEmpty()) {
- env.put(envName,entry.getValue());
+ env.put(envName, entry.getValue());
}
}
}
@@ -863,8 +902,8 @@ public final class SliderUtils {
* @param commandOptions command opts
*/
public static void applyCommandLineRoleOptsToRoleMap(Map<String, Map<String, String>> clusterRoleMap,
- Map<String, Map<String, String>> commandOptions) {
- for (Map.Entry<String, Map<String, String>> entry: commandOptions.entrySet()) {
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
String key = entry.getKey();
Map<String, String> optionMap = entry.getValue();
Map<String, String> existingMap = clusterRoleMap.get(key);
@@ -872,7 +911,7 @@ public final class SliderUtils {
existingMap = new HashMap<String, String>();
}
log.debug("Overwriting role options with command line values {}",
- stringifyMap(optionMap));
+ stringifyMap(optionMap));
mergeMap(existingMap, optionMap);
//set or overwrite the role
clusterRoleMap.put(key, existingMap);
@@ -885,10 +924,10 @@ public final class SliderUtils {
* @throws BadCommandArgumentsException if it is invalid
*/
public static void validateClusterName(String clustername) throws
- BadCommandArgumentsException {
+ BadCommandArgumentsException {
if (!isClusternameValid(clustername)) {
throw new BadCommandArgumentsException(
- "Illegal cluster name: " + clustername);
+ "Illegal cluster name: " + clustername);
}
}
@@ -900,12 +939,12 @@ public final class SliderUtils {
* @throws BadConfigException if the key is not set
*/
public static void verifyPrincipalSet(Configuration conf,
- String principal) throws
- BadConfigException {
+ String principal) throws
+ BadConfigException {
String principalName = conf.get(principal);
if (principalName == null) {
throw new BadConfigException("Unset Kerberos principal : %s",
- principal);
+ principal);
}
log.debug("Kerberos princial {}={}", principal, principalName);
}
@@ -928,8 +967,8 @@ public final class SliderUtils {
* @throws BadConfigException the configuration/process is invalid
*/
public static boolean maybeInitSecurity(Configuration conf) throws
- IOException,
- BadConfigException {
+ IOException,
+ BadConfigException {
boolean clusterSecure = isHadoopClusterSecure(conf);
if (clusterSecure) {
log.debug("Enabling security");
@@ -946,8 +985,8 @@ public final class SliderUtils {
* @throws BadConfigException the configuration and system state are inconsistent
*/
public static boolean initProcessSecurity(Configuration conf) throws
- IOException,
- BadConfigException {
+ IOException,
+ BadConfigException {
if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
//security is already inited
@@ -959,9 +998,9 @@ public final class SliderUtils {
//this gets UGI to reset its previous world view (i.e simple auth)
//security
log.debug("java.security.krb5.realm={}",
- System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
log.debug("java.security.krb5.kdc={}",
- System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
log.debug("hadoop.security.authentication={}",
conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
log.debug("hadoop.security.authorization={}",
@@ -974,14 +1013,14 @@ public final class SliderUtils {
log.debug("Login user is {}", UserGroupInformation.getLoginUser());
if (!UserGroupInformation.isSecurityEnabled()) {
throw new BadConfigException("Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
- authUser);
+ "the application has already set up its user as an insecure entity %s",
+ authUser);
}
if (authUser.getAuthenticationMethod() ==
UserGroupInformation.AuthenticationMethod.SIMPLE) {
throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method",
- authUser);
+ " -security has already been set up with the wrong authentication method",
+ authUser);
}
@@ -1017,24 +1056,25 @@ public final class SliderUtils {
* @throws IOException trouble copying to HDFS
*/
public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/"+ jarName, res);
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
return res;
}
- public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>();
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest =
+ new HashMap<String, Map<String, String>>();
for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
dest.put(entry.getKey(), stringMapClone(entry.getValue()));
}
@@ -1042,7 +1082,7 @@ public final class SliderUtils {
}
public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<String, String>();
+ Map<String, String> dest = new HashMap<String, String>();
return mergeEntries(dest, src.entrySet());
}
@@ -1146,12 +1186,12 @@ public final class SliderUtils {
* @return a classpath
*/
public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- Configuration config,
- boolean usingMiniMRCluster) {
+ String libdir,
+ Configuration config,
+ boolean usingMiniMRCluster) {
ClasspathConstructor classpath = new ClasspathConstructor();
-
+
// add the runtime classpath needed for tests to work
if (usingMiniMRCluster) {
// for mini cluster we pass down the java CP properties
@@ -1175,17 +1215,18 @@ public final class SliderUtils {
* @param errorlog log for output on an error
* @throws FileNotFoundException if it is not a directory
*/
- public static void verifyIsDir(File dir, Logger errorlog) throws FileNotFoundException {
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
if (!dir.exists()) {
errorlog.warn("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
+ listDir(dir.getParentFile()));
throw new FileNotFoundException(dir.toString());
}
if (!dir.isDirectory()) {
errorlog.info("contents of {}: {}", dir,
- listDir(dir.getParentFile()));
+ listDir(dir.getParentFile()));
throw new FileNotFoundException(
- "Not a directory: " + dir);
+ "Not a directory: " + dir);
}
}
@@ -1195,10 +1236,11 @@ public final class SliderUtils {
* @param errorlog log for output on an error
* @throws FileNotFoundException
*/
- public static void verifyFileExists(File file, Logger errorlog) throws FileNotFoundException {
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
if (!file.exists()) {
errorlog.warn("contents of {}: {}", file,
- listDir(file.getParentFile()));
+ listDir(file.getParentFile()));
throw new FileNotFoundException(file.toString());
}
if (!file.isFile()) {
@@ -1214,15 +1256,15 @@ public final class SliderUtils {
* @throws BadConfigException if the key is missing
*/
public static String verifyOptionSet(Configuration configuration, String key,
- boolean allowEmpty) throws BadConfigException {
+ boolean allowEmpty) throws BadConfigException {
String val = configuration.get(key);
if (val == null) {
throw new BadConfigException(
- "Required configuration option \"%s\" not defined ", key);
+ "Required configuration option \"%s\" not defined ", key);
}
if (!allowEmpty && val.isEmpty()) {
throw new BadConfigException(
- "Configuration option \"%s\" must not be empty", key);
+ "Configuration option \"%s\" must not be empty", key);
}
return val;
}
@@ -1235,24 +1277,25 @@ public final class SliderUtils {
* @return the file referenced
* @throws BadConfigException on a failure
*/
- public static File verifyKeytabExists(Configuration siteConf, String prop) throws
- BadConfigException {
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
String keytab = siteConf.get(prop);
if (keytab == null) {
throw new BadConfigException("Missing keytab property %s",
- prop);
+ prop);
}
File keytabFile = new File(keytab);
if (!keytabFile.exists()) {
throw new BadConfigException("Missing keytab file %s defined in %s",
- keytabFile,
- prop);
+ keytabFile,
+ prop);
}
if (keytabFile.length() == 0 || !keytabFile.isFile()) {
throw new BadConfigException("Invalid keytab file %s defined in %s",
- keytabFile,
- prop);
+ keytabFile,
+ prop);
}
return keytabFile;
}
@@ -1278,12 +1321,12 @@ public final class SliderUtils {
Properties props = SliderVersionInfo.loadVersionProperties();
info.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO, props.getProperty(
- SliderVersionInfo.APP_BUILD_INFO));
+ SliderVersionInfo.APP_BUILD_INFO));
info.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO,
- props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
+ props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO));
info.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO,
- VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
+ VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum());
}
/**
@@ -1295,30 +1338,33 @@ public final class SliderUtils {
* @param time timestamp
*/
public static void setInfoTime(Map info,
- String keyHumanTime,
- String keyMachineTime,
- long time) {
+ String keyHumanTime,
+ String keyMachineTime,
+ long time) {
info.put(keyHumanTime, SliderUtils.toGMTString(time));
info.put(keyMachineTime, Long.toString(time));
}
- public static Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws
+ public static Path extractImagePath(CoreFileSystem fs,
+ MapOperations internalOptions) throws
SliderException, IOException {
Path imagePath;
String imagePathOption =
internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
+ String appHomeOption =
+ internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
imagePath = fs.createPathThatMustExist(imagePathOption);
} else {
imagePath = null;
if (isUnset(appHomeOption)) {
- throw new BadClusterStateException(ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
+ throw new BadClusterStateException(
+ ErrorStrings.E_NO_IMAGE_OR_HOME_DIR_SPECIFIED);
}
}
return imagePath;
}
-
+
/**
* trigger a JVM halt with no clean shutdown at all
* @param status status code for exit
@@ -1373,7 +1419,7 @@ public final class SliderUtils {
* @param paths subpaths
* @return base+"/"+paths[0]+"/"+paths[1]...
*/
- public static String appendToURL(String base, String...paths) {
+ public static String appendToURL(String base, String... paths) {
String result = base;
for (String path : paths) {
result = appendToURL(result, path);
@@ -1390,19 +1436,19 @@ public final class SliderUtils {
* @return
*/
public static String truncate(String toTruncate, int maxSize) {
- if(toTruncate == null || maxSize < 1
- || toTruncate.length() <= maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
return toTruncate;
}
String pad = "...";
- if(maxSize < 10) {
+ if (maxSize < 10) {
pad = "";
}
return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
}
-
-
+
+
/**
* Callable for async/scheduled halt
*/
@@ -1436,11 +1482,15 @@ public final class SliderUtils {
*/
public static int compareTo(long left, long right) {
long diff = left - right;
- if (diff < 0) return -1;
- if (diff > 0) return 1;
+ if (diff < 0) {
+ return -1;
+ }
+ if (diff > 0) {
+ return 1;
+ }
return 0;
}
-
+
/**
* This wrapps ApplicationReports and generates a string version
* iff the toString() operator is invoked
@@ -1459,8 +1509,8 @@ public final class SliderUtils {
}
public static InputStream getApplicationResourceInputStream(FileSystem fs,
- Path appPath,
- String entry)
+ Path appPath,
+ String entry)
throws IOException {
InputStream is = null;
FSDataInputStream appStream = null;
@@ -1473,7 +1523,8 @@ public final class SliderUtils {
if (entry.equals(zipEntry.getName())) {
int size = (int) zipEntry.getSize();
if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(), zipEntry.getSize());
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
byte[] content = new byte[size];
int offset = 0;
while (offset < size) {
@@ -1509,42 +1560,218 @@ public final class SliderUtils {
* the headers.
* @throws IOException on any problem reading the file
* @throws FileNotFoundException if the file is not considered valid
+ * @param logger
*/
- public static void verifyWinUtilsValid() throws IOException {
+ public static void maybeVerifyWinUtilsValid(Logger logger) throws IOException, SliderException {
if (!Shell.WINDOWS) {
return;
}
- String winUtilsPath = Shell.getWinUtilsPath();
- if (winUtilsPath == null) {
- throw new FileNotFoundException(WINUTILS + " not found on Path : " +
+ String exePath = Shell.getWinUtilsPath();
+ String program = WINUTILS;
+ if (exePath == null) {
+ throw new FileNotFoundException(program + " not found on Path : " +
System.getenv("Path"));
}
- File winUtils = new File(winUtilsPath);
- if (!winUtils.isFile()) {
- throw new FileNotFoundException(WINUTILS
- + " at " + winUtilsPath
- + " is not a file");
+ File exe = new File(exePath);
+
+ verifyWindowsExe(program, exe);
+ execCommand(WINUTILS, 0, 5000, log, null, exePath, "systeminfo");
+
+ }
+
+ protected static void verifyIsFile(String program, File exe) throws
+ FileNotFoundException {
+ if (!exe.isFile()) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is not a file");
}
- if (winUtils.length() < 0x100) {
- throw new FileNotFoundException(WINUTILS
- + " at " + winUtilsPath
- + " is too short to be an executable");
+ }
+
+ protected static void verifyFileSize(String program,
+ File exe,
+ int minFileSize) throws FileNotFoundException {
+ if (exe.length() < minFileSize) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is too short to be an executable");
}
+ }
+
+ /**
+ * Look for the windows executable and check it has the right headers.
+ * <code>File.canRead()</code> doesn't work on windows, so the reading
+ * is mandatory.
+ *
+ * @param program program name for errors
+ * @param exe executable
+ * @throws IOException IOE
+ */
+ public static void verifyWindowsExe(String program, File exe)
+ throws IOException {
+ verifyIsFile(program, exe);
+
+ verifyFileSize(program, exe, 0x100);
+
// now read two bytes and verify the header.
+
FileReader reader = null;
try {
int[] header = new int[2];
- reader = new FileReader(winUtilsPath);
+ reader = new FileReader(exe);
header[0] = reader.read();
header[1] = reader.read();
- if (header[0] != 'M' || header[1] != 'Z') {
- throw new FileNotFoundException(WINUTILS
- + " at " + winUtilsPath
- + " is not a windows executable file");
+ if ((header[0] != 'M' || header[1] != 'Z')) {
+ throw new FileNotFoundException(program
+ + " at " + exe
+ + " is not a windows executable file");
}
} finally {
IOUtils.closeStream(reader);
}
}
+
+ /**
+ * Verify that a Unix exe works
+ * @param program program name for errors
+ * @param exe executable
+ * @throws IOException IOE
+
+ */
+ public static void verifyUnixExe(String program, File exe)
+ throws IOException {
+ verifyIsFile(program, exe);
+
+ // read flag
+ if (!exe.canRead()) {
+ throw new IOException("Cannot read " + program + " at " + exe);
+ }
+ // exe flag
+ if (!exe.canExecute()) {
+ throw new IOException("Cannot execute " + program + " at " + exe);
+ }
+ }
+
+ /**
+ * Validate an executable
+ * @param program
+ * @param exe
+ * @throws IOException
+ */
+ public static void validateExe(String program, File exe) throws IOException {
+ if (!Shell.WINDOWS) {
+ verifyWindowsExe(program, exe);
+ } else {
+ verifyUnixExe(program, exe);
+ }
+ }
+
+
+ /**
+ * Execute a command for a test operation
+ * @param name name in error
+ * @param status status code expected
+ * @param timeoutMillis timeout in millis for process to finish
+ * @param logger
+ *@param outputString optional string to grep for (must not span a line)
+ * @param commands commands @return the process
+ * @throws IOException on any failure.
+ */
+ public static ForkedProcessService execCommand(String name,
+ int status,
+ long timeoutMillis,
+ Logger logger,
+ String outputString,
+ String... commands) throws IOException, SliderException {
+ Preconditions.checkArgument(isSet(name), "no name");
+ Preconditions.checkArgument(commands.length > 0, "no commands");
+ Preconditions.checkArgument(isSet(commands[0]), "empty command");
+
+ ForkedProcessService process;
+
+
+ process = new ForkedProcessService(
+ name,
+ new HashMap<String, String>(),
+ Arrays.asList(commands));
+ process.setProcessLog(logger);
+ process.init(new Configuration());
+ String errorText = null;
+ EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
+ process.start();
+ try {
+ waiter.waitForServiceToStop(timeoutMillis);
+ int exitCode = process.getExitCode();
+ List<String> recentOutput = process.getRecentOutput();
+ if (status != exitCode) {
+ // error condition
+ errorText = "Expected exit code={" + status + "}, "
+ + "actual exit code={" + exitCode + "}";
+ } else {
+ if (isSet(outputString)) {
+ boolean found = false;
+ for (String line : recentOutput) {
+ if (line.contains(outputString)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ errorText = "Did not find \"" + outputString + "\""
+ + " in output";
+ }
+ }
+ }
+ if (errorText== null) {
+ return process;
+ }
+
+ } catch (InterruptedException e) {
+ throw new InterruptedIOException(e.toString());
+ } catch (TimeoutException e) {
+ log.debug("");
+ errorText = e.toString();
+ }
+ // error text: non null ==> operation failed
+ log.warn(errorText);
+ List<String> recentOutput = process.getRecentOutput();
+ for (String line : recentOutput) {
+ log.info(line);
+ }
+ throw new SliderException(LauncherExitCodes.EXIT_OTHER_FAILURE,
+ "Process %s failed: %s", name, errorText);
+
+ }
+
+
+ /**
+ * Validate the slider client-side execution environment.
+ * This looks for everything felt to be critical for execution, including
+ * native binaries and other essential dependencies.
+ * @param logger logger to log to on normal execution
+ * @throws IOException on IO failures
+ * @throws SliderException on validation failures
+ */
+ public static void validateSliderClientEnvironment(Logger logger) throws
+ IOException,
+ SliderException {
+ maybeVerifyWinUtilsValid(logger);
+ }
+
+ /**
+ * Validate the slider server-side execution environment.
+ * This looks for everything felt to be critical for execution, including
+ * native binaries and other essential dependencies.
+ * @param logger logger to log to on normal execution
+ * @throws IOException on IO failures
+ * @throws SliderException on validation failures
+ */
+ public static void validateSliderServerEnvironment(Logger logger) throws
+ IOException,
+ SliderException {
+ maybeVerifyWinUtilsValid(logger);
+ execCommand(OPENSSL, 0, 5000, logger, "OpenSSL", OPENSSL, "version");
+ execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "--version");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
new file mode 100644
index 0000000..6e8add1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/EndOfServiceWaiter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.utility;
+
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wait for a service to stop.
+ *
+ * WARNING: the notification may come in as soon as the service enters
+ * the stopped state: it may take some time for the actual stop operation
+ * to complete.
+ */
+public class EndOfServiceWaiter implements ServiceStateChangeListener {
+
+ private final AtomicBoolean finished = new AtomicBoolean(false);
+ private final String name;
+
+ /**
+ * Wait for a service; use the service name as this instance's name
+ * @param service service
+ */
+ public EndOfServiceWaiter(Service service) {
+ this(service.getName(), service);
+ }
+
+
+ /**
+ * Wait for a service
+ * @param name name for messages
+ * @param service service
+ */
+ public EndOfServiceWaiter(String name, Service service) {
+ this.name = name;
+ service.registerServiceListener(this);
+ }
+
+ public synchronized void waitForServiceToStop(long timeout) throws
+ InterruptedException, TimeoutException {
+ if (!finished.get()) {
+ wait(timeout);
+ }
+ if (!finished.get()) {
+ throw new TimeoutException(name
+ + " did not finish after " + timeout +
+ " milliseconds");
+ }
+ }
+
+ /**
+ * Wait for service state change callbacks; notify self if the service has
+ * now stopped
+ * @param service service
+ */
+ @Override
+ public synchronized void stateChanged(Service service) {
+ if (service.isInState(Service.STATE.STOPPED)) {
+ finished.set(true);
+ notify();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
index 46c724c..b801993 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -81,7 +81,11 @@ public class ForkedProcessService
private LongLivedProcess process;
private int executionTimeout = -1;
private int timeoutCode = 1;
-
+ /**
+ log to log to; defaults to this service log
+ */
+ private Logger processLog = LOG;
+
/**
* Exit code set when the spawned process exits
*/
@@ -102,7 +106,8 @@ public class ForkedProcessService
* @param env environment variables above those generated by
* @throws IOException IO problems
*/
- public ForkedProcessService(String name, Map<String, String> env,
+ public ForkedProcessService(String name,
+ Map<String, String> env,
List<String> commandList) throws IOException {
super(name);
build(env, commandList);
@@ -130,6 +135,15 @@ public class ForkedProcessService
}
/**
+ * Set the process log. This may be null for "do not log"
+ * @param processLog process log
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ process.setProcessLog(processLog);
+ }
+
+ /**
* Set the timeout by which time a process must have finished -or -1 for forever
* @param timeout timeout in milliseconds
*/
@@ -148,7 +162,8 @@ public class ForkedProcessService
List<String> commandList)
throws IOException {
assert process == null;
- process = new LongLivedProcess(getName(), LOG, commandList);
+
+ process = new LongLivedProcess(getName(), processLog, commandList);
process.setLifecycleCallback(this);
//set the env variable mapping
process.putEnvMap(env);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index c8ff758..6f83811 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -90,7 +90,7 @@ public class LongLivedProcess implements Runnable {
* Log supplied in the constructor for the spawned process -accessible
* to inner classes
*/
- private final Logger processLog;
+ private Logger processLog;
/**
* Class log -accessible to inner classes
@@ -102,10 +102,15 @@ public class LongLivedProcess implements Runnable {
*/
private final AtomicBoolean finished = new AtomicBoolean(false);
+ /**
+ * Create an instance
+ * @param name process name
+ * @param processLog log for output (or null)
+ * @param commands command list
+ */
public LongLivedProcess(String name,
Logger processLog,
List<String> commands) {
- Preconditions.checkArgument(processLog != null, "processLog");
Preconditions.checkArgument(commands != null, "commands");
this.name = name;
@@ -168,6 +173,14 @@ public class LongLivedProcess implements Runnable {
}
/**
+ * Set the process log. Ignored once the process starts
+ * @param processLog new log ... may be null
+ */
+ public void setProcessLog(Logger processLog) {
+ this.processLog = processLog;
+ }
+
+ /**
* Get the process reference
* @return the process -null if the process is not started
*/
@@ -398,10 +411,11 @@ public class LongLivedProcess implements Runnable {
* something that is only called once per line of IO?
* @param line line to record
* @param isErrorStream is the line from the error stream
- * @param logger logger to log to
+ * @param logger logger to log to - null for no logging
*/
private synchronized void recordRecentLine(String line,
- boolean isErrorStream, Logger logger) {
+ boolean isErrorStream,
+ Logger logger) {
if (line == null) {
return;
}
@@ -410,10 +424,12 @@ public class LongLivedProcess implements Runnable {
if (recentLines.size() > recentLineLimit) {
recentLines.remove(0);
}
- if (isErrorStream) {
- logger.warn(line);
- } else {
- logger.info(line);
+ if (logger != null) {
+ if (isErrorStream) {
+ logger.warn(line);
+ } else {
+ logger.info(line);
+ }
}
}
@@ -428,6 +444,12 @@ public class LongLivedProcess implements Runnable {
private final Logger streamLog;
private final int sleepTime;
+ /**
+ * Create an instance
+ * @param streamLog log -or null to disable logging (recent entries
+ * will still be retained)
+ * @param sleepTime time to sleep when stopping
+ */
private ProcessStreamReader(Logger streamLog, int sleepTime) {
this.streamLog = streamLog;
this.sleepTime = sleepTime;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
new file mode 100644
index 0000000..5489366
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestExecutionEnvironment extends SliderTestBase {
+
+ @Test
+ public void testClientEnv() throws Throwable {
+ SliderUtils.validateSliderClientEnvironment(log)
+ }
+
+
+ @Test
+ public void testServerEnv() throws Throwable {
+ SliderUtils.validateSliderServerEnvironment(log)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 6ae683a..98fa183 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -27,7 +27,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
import org.apache.slider.providers.agent.AgentUtils
-import org.apache.slider.server.services.workflow.EndOfServiceWaiter
+import org.apache.slider.server.services.utility.EndOfServiceWaiter
import org.apache.slider.server.services.workflow.ForkedProcessService
import org.apache.slider.test.SliderTestBase
import org.junit.Test
@@ -134,7 +134,7 @@ class TestWindowsSupport extends SliderTestBase {
@Test
public void testHasWinutils() throws Throwable {
assume(Shell.WINDOWS, "not windows")
- SliderUtils.verifyWinUtilsValid()
+ SliderUtils.maybeVerifyWinUtilsValid(log)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
deleted file mode 100644
index 5e6df3b..0000000
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/EndOfServiceWaiter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.junit.Assert;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-
-/**
- * Wait for a service to stop
- */
-public class EndOfServiceWaiter implements ServiceStateChangeListener {
-
- private final AtomicBoolean finished = new AtomicBoolean(false);
-
- public EndOfServiceWaiter(Service svc) {
- svc.registerServiceListener(this);
- }
-
- public synchronized void waitForServiceToStop(long timeout) throws
- InterruptedException {
- if (!finished.get()) {
- wait(timeout);
- }
- Assert.assertTrue("Service did not finish in time period",
- finished.get());
- }
-
- @Override
- public synchronized void stateChanged(Service service) {
- if (service.isInState(Service.STATE.STOPPED)) {
- finished.set(true);
- notify();
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0f67b104/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
index 6d08156..e8f7d88 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
@@ -20,6 +20,7 @@ package org.apache.slider.server.services.workflow;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.ServiceOperations;
+import org.apache.slider.server.services.utility.EndOfServiceWaiter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -31,6 +32,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeoutException;
/**
* Test the long lived process by executing a command that works and a command
@@ -131,7 +133,7 @@ public class TestWorkflowForkedProcessService extends WorkflowServiceTestBase {
return process;
}
- public void exec() throws InterruptedException {
+ public void exec() throws InterruptedException, TimeoutException {
assertNotNull(process);
EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
process.start();
[18/47] git commit: SLIDER-377 switch to branch 2.6 to see if
failures arise
Posted by st...@apache.org.
SLIDER-377 switch to branch 2.6 to see if failures arise
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/877a0160
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/877a0160
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/877a0160
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 877a01609748546a021d5477b7cd52f7f1536c65
Parents: 93c3c77
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 14:11:29 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 14:11:29 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/877a0160/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 454918c..8292073 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,7 +118,7 @@
<!--
core artifacts
-->
- <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
<accumulo.version>1.6.1-SNAPSHOT</accumulo.version>
[46/47] git commit: Merge branch 'develop' into
feature/SLIDER-149_Support_a_YARN_service_registry
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-149_Support_a_YARN_service_registry
Conflicts:
slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/901a6738
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/901a6738
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/901a6738
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 901a67385d6c7bce6cf8e47c58590282985b684c
Parents: 6e883bf f14d32c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 14:17:04 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 14:17:04 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 7 +-
.../src/test/resources/appConfig_kerberos.json | 57 ++
.../test/resources/appConfig_monitor_ssl.json | 7 +-
.../src/test/resources/appConfig_ssl.json | 7 +-
app-packages/hbase/appConfig.json | 8 +-
app-packages/storm/README.txt | 19 +-
app-packages/storm/appConfig.json | 16 +-
app-packages/storm/configuration/storm-env.xml | 25 +
app-packages/storm/metainfo.xml | 26 +-
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 -
app-packages/storm/package/scripts/params.py | 15 +-
app-packages/storm/package/scripts/service.py | 8 +
.../storm/package/scripts/status_params.py | 4 +-
.../storm/package/templates/config.yaml.j2 | 9 -
.../storm/package/templates/storm_jaas.conf.j2 | 27 +-
app-packages/storm/pom.xml | 90 +++
app-packages/storm/resources.json | 10 +-
app-packages/storm/src/assembly/storm.xml | 68 +++
pom.xml | 42 +-
slider-agent/pom.xml | 4 +-
slider-agent/src/main/python/setup.py | 2 +-
slider-agent/src/test/python/python-wrap | 40 --
slider-agent/src/test/python/python-wrap.sh | 42 ++
slider-agent/src/test/python/unitTests.py | 36 +-
slider-core/pom.xml | 4 +-
.../org/apache/slider/api/InternalKeys.java | 11 +
.../org/apache/slider/api/ResourceKeys.java | 13 -
.../org/apache/slider/client/SliderClient.java | 33 +-
.../common/params/AbstractActionArgs.java | 11 +
.../slider/common/params/ActionHelpArgs.java | 14 +-
.../slider/common/params/ActionVersionArgs.java | 12 +-
.../apache/slider/common/tools/SliderUtils.java | 571 ++++++++++++++-----
.../slider/core/build/InstanceBuilder.java | 5 +-
.../apache/slider/core/zk/ZookeeperUtils.java | 3 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 38 +-
.../server/appmaster/RoleLaunchService.java | 8 +-
.../server/appmaster/SliderAppMaster.java | 68 ++-
.../server/appmaster/actions/ActionHalt.java | 6 +-
.../appmaster/actions/ActionKillContainer.java | 20 +-
.../appmaster/actions/ActionStartContainer.java | 8 +-
.../appmaster/actions/ActionStopSlider.java | 6 +-
.../server/appmaster/actions/AsyncAction.java | 47 +-
.../actions/ProviderReportedContainerLoss.java | 13 +-
.../actions/ProviderStartupCompleted.java | 4 -
.../server/appmaster/actions/QueueAccess.java | 5 +
.../server/appmaster/actions/QueueService.java | 11 +-
.../actions/RegisterComponentInstance.java | 5 +
.../actions/ReviewAndFlexApplicationSize.java | 43 ++
.../actions/UnregisterComponentInstance.java | 7 +-
.../server/appmaster/monkey/ChaosEntry.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 11 +-
.../services/utility/EndOfServiceWaiter.java | 84 +++
.../services/workflow/ForkedProcessService.java | 21 +-
.../services/workflow/LongLivedProcess.java | 57 +-
.../agent/AgentMiniClusterTestBase.groovy | 9 +
.../agent/actions/TestActionVersion.groovy | 5 +-
.../standalone/TestStandaloneAMKill.groovy | 2 +-
.../TestStandaloneAMMonkeyRestart.groovy | 8 +-
.../tools/TestClientResourceRegistration.groovy | 3 +
.../common/tools/TestConfigHelperHDFS.groovy | 3 +
.../tools/TestExecutionEnvironment.groovy | 42 ++
.../common/tools/TestMiscSliderUtils.groovy | 3 +
.../common/tools/TestWindowsSupport.groovy | 76 ++-
.../common/tools/TestZKIntegration.groovy | 54 +-
.../slider/providers/agent/AgentTestBase.groovy | 10 +
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 31 +-
.../test/YarnZKMiniClusterTestBase.groovy | 2 +-
.../agent/TestAgentProviderService.java | 29 +
.../services/workflow/EndOfServiceWaiter.java | 56 --
.../TestWorkflowForkedProcessService.java | 4 +-
.../providers/accumulo/AccumuloTestBase.groovy | 4 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 1 -
.../flexing/TestClusterFlex1To1.groovy | 2 +-
75 files changed, 1509 insertions(+), 564 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
index 61bf5dd,543c1a8..3145ecb
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@@ -25,14 -25,17 +25,19 @@@ import org.apache.slider.server.appmast
import java.util.concurrent.TimeUnit;
+ /**
+ * Notify the app master that it should register a component instance
+ * in the registry
+ * {@link SliderAppMaster#registerComponent(ContainerId)}
+ */
public class RegisterComponentInstance extends AsyncAction {
-
public final ContainerId containerId;
+ public final String description;
- public RegisterComponentInstance(ContainerId containerId, long delay,
+ public RegisterComponentInstance(ContainerId containerId,
+ String description,
+ long delay,
TimeUnit timeUnit) {
super("RegisterComponentInstance :" + containerId,
delay, timeUnit);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --cc slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 3dc1d6d,97cc853..d80ee62
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@@ -246,9 -244,12 +243,11 @@@ public abstract class YarnMiniClusterTe
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
name = buildClustername(name)
- miniCluster =
- new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs, 1, false, false)
- miniCluster.init(conf)
+ miniCluster = new MiniYARNCluster(
+ name,
+ noOfNodeManagers,
+ numLocalDirs,
+ numLogDirs)
- miniCluster.init(conf)
miniCluster.start();
if (startHDFS) {
createMiniHDFSCluster(name, conf)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/901a6738/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
[19/47] git commit: Merge branch 'feature/SLIDER-377-windows' into
develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-377-windows' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/29a40f50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/29a40f50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/29a40f50
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 29a40f5094b4275fa4d7159d91f1c965cdc6da50
Parents: 38c816e 877a016
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 14:41:54 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 14:41:54 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 50 ++++++++++++-
.../common/tools/TestWindowsSupport.groovy | 76 +++++++++++++++++++-
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
3 files changed, 123 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[30/47] git commit: SLIDER-393 fix pom to publish jars to ASF staging
and snapshots (Giridharan Kesavan via billie)
Posted by st...@apache.org.
SLIDER-393 fix pom to publish jars to ASF staging and snapshots (Giridharan Kesavan via billie)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6126db62
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6126db62
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6126db62
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6126db62bb5d4ae45b6ecddb7b71ce1b8ae13cb2
Parents: a0a3ae6
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu Sep 4 10:40:03 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Sep 4 10:40:03 2014 -0700
----------------------------------------------------------------------
pom.xml | 41 +++++++++++++++++++++++++++++++++++++++--
1 file changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6126db62/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8292073..9efb5c8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -72,6 +72,19 @@
<url>http://slider.incubator.apache.org/</url>
</site>
<downloadUrl>http://git-wip-us.apache.org/repos/asf/incubator-slider.git</downloadUrl>
+ <repository>
+ <id>${distMgmtStagingId}</id>
+ <name>${distMgmtStagingName}</name>
+ <url>${distMgmtStagingUrl}</url>
+ </repository>
+ <snapshotRepository>
+ <id>${distMgmtSnapshotsId}</id>
+ <name>${distMgmtSnapshotsName}</name>
+ <url>${distMgmtSnapshotsUrl}</url>
+ </snapshotRepository>
+
+
+
</distributionManagement>
<mailingLists>
@@ -92,7 +105,12 @@
</mailingLists>
<properties>
-
+ <distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>
+ <distMgmtSnapshotsName>Apache Development Snapshot Repository</distMgmtSnapshotsName>
+ <distMgmtSnapshotsUrl>https://repository.apache.org/content/repositories/snapshots</distMgmtSnapshotsUrl>
+ <distMgmtStagingId>apache.staging.https</distMgmtStagingId>
+ <distMgmtStagingName>Apache Release Distribution Repository</distMgmtStagingName>
+ <distMgmtStagingUrl>https://repository.apache.org/service/local/staging/deploy/maven2</distMgmtStagingUrl>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -1397,7 +1415,26 @@
</build>
</profile>
-
+ <profile>
+ <id>sign</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>sign-artifacts</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>sign</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
[36/47] git commit: SLIDER-397 taskkill /f
Posted by st...@apache.org.
SLIDER-397 taskkill /f
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ee3ad77a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ee3ad77a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ee3ad77a
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: ee3ad77a83f277e42ecef39b30fd614d66cc5530
Parents: 27f0a62
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 20:43:15 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 20:43:15 2014 +0100
----------------------------------------------------------------------
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ee3ad77a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 5086248..dbf5c49 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID "
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /f /PID\""
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[28/47] git commit: SLIDER-377 tuning of some tests & making a couple
of methods accessible (groovy was apparently bending the rules)
Posted by st...@apache.org.
SLIDER-377 tuning of some tests & making a couple of methods accessible (groovy was apparently bending the rules)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4fc0396e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4fc0396e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4fc0396e
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 4fc0396e8705b0e50bbc6bd9671c1b8b5dbb3a0a
Parents: f2965d8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 18:19:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 18:19:01 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 6 ++-
.../agent/actions/TestActionVersion.groovy | 5 +--
.../common/tools/TestZKIntegration.groovy | 44 ++++++++++----------
.../slider/providers/agent/AgentTestBase.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 27 ++++++------
5 files changed, 43 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fc0396e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f4040c4..ef6448d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -355,7 +355,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
- protected boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
+ @VisibleForTesting
+ public boolean deleteZookeeperNode(String clusterName) throws YarnException, IOException {
String user = getUsername();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
Exception e = null;
@@ -391,7 +392,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Create the zookeeper node associated with the calling user and the cluster
*/
- protected String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
+ @VisibleForTesting
+ public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
String user = getUsername();
String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
if(nameOnly) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fc0396e/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
index 1d50b71..7a2820f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionVersion.groovy
@@ -20,10 +20,9 @@ package org.apache.slider.agent.actions
import groovy.util.logging.Slf4j
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.YarnMiniClusterTestBase
+import org.apache.slider.test.SliderTestBase
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Before
import org.junit.Test
/**
@@ -31,7 +30,7 @@ import org.junit.Test
*/
@Slf4j
-class TestActionVersion extends YarnMiniClusterTestBase {
+class TestActionVersion extends SliderTestBase {
@Test
public void testVersion() throws Throwable {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fc0396e/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index fe3bef7..c13af78 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.slider.client.SliderClient
@@ -31,12 +32,16 @@ import org.junit.Before
import org.junit.Test
@Slf4j
-
+@CompileStatic
class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTests {
+
+ // as the static compiler doesn't resolve consistently
+ public static final String USER = KeysForTests.USERNAME
+
@Before
void createCluster() {
- Configuration conf = getConfiguration()
+ Configuration conf = configuration
createMicroZKCluster(conf)
}
@@ -44,7 +49,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
public void testIntegrationCreate() throws Throwable {
assertHasZKCluster()
ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "cluster1", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
Stat stat = zki.stat(userPath)
assert stat != null
log.info("User path $userPath has stat $stat")
@@ -55,7 +60,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
assertHasZKCluster()
ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
List<String> clusters = zki.clusters
assert clusters.empty
}
@@ -65,7 +70,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
assertHasZKCluster()
ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
String fullPath = zki.createPath(userPath, "/cluster-",
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL_SEQUENTIAL)
@@ -78,7 +83,7 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Test
public void testListUserClustersWithTwoCluster() throws Throwable {
ZKIntegration zki = createZKIntegrationInstance(ZKBinding, "", true, false, 5000)
- String userPath = ZKIntegration.mkSliderUserPath(USERNAME)
+ String userPath = ZKIntegration.mkSliderUserPath(USER)
String c1 = createEphemeralChild(zki, userPath)
log.info("Ephemeral path $c1")
String c2 = createEphemeralChild(zki, userPath)
@@ -94,25 +99,24 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
MockSliderClient client = new MockSliderClient()
String path = client.createZookeeperNode("cl1", true)
- ZKIntegration zki = client.getLastZKIntegration()
+ ZKIntegration zki = client.lastZKIntegration
- String zkPath = ZKIntegration.mkClusterPath(USERNAME, "cl1")
- assert zkPath == "/services/slider/users/" + USERNAME + "/cl1", "zkPath must be as expected"
+ String zkPath = ZKIntegration.mkClusterPath(USER, "cl1")
+ assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
assert zki == null, "ZKIntegration should be null."
zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000);
- assert false == zki.exists(zkPath), "zkPath should not exist"
+ assert !zki.exists(zkPath)
path = client.createZookeeperNode("cl1", false)
- zki = client.getLastZKIntegration()
- assert zkPath == "/services/slider/users/" + USERNAME + "/cl1", "zkPath must be as expected"
+ zki = client.lastZKIntegration
+ assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
- assert true == zki.exists(zkPath), "zkPath must exist"
+ assert zki.exists(zkPath)
zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT)
- assert true == zki.exists(zkPath + "/cn"), "zkPath with child node must exist"
+ assert zki.exists(zkPath + "/cn")
client.deleteZookeeperNode("cl1")
- assert false == zki.exists(zkPath), "zkPath must not exist"
-
+ assert !zki.exists(zkPath)
}
public String createEphemeralChild(ZKIntegration zki, String userPath) {
@@ -126,19 +130,17 @@ class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTest
@Override
public String getUsername() {
- return USERNAME
+ return USER
}
@Override
protected ZKIntegration getZkClient(String clusterName, String user) {
- zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000)
- return zki;
+ createZKIntegrationInstance(ZKBinding, "cl1", true, false, 5000)
}
@Override
public synchronized Configuration getConfig() {
- Configuration conf = new Configuration();
- return conf;
+ new Configuration();
}
public ZKIntegration getLastZKIntegration() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fc0396e/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 2953381..ff67a71 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -53,7 +53,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
* Server side test: validate system env before launch
*/
@BeforeClass
- public static void checkSystem() {
+ public static void checkServerEnv() {
SliderUtils.validateSliderServerEnvironment(null)
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fc0396e/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index f1fd6de..7ffa64a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -49,11 +49,8 @@ import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.apache.slider.server.appmaster.SliderAppMaster
import org.junit.After
-import org.junit.Assert
-import org.junit.Before
import org.junit.BeforeClass
import org.junit.Rule
-import org.junit.rules.TestName
import org.junit.rules.Timeout
import static org.apache.slider.test.KeysForTests.*
@@ -129,17 +126,14 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
KEY_TEST_TIMEOUT,
DEFAULT_TEST_TIMEOUT_SECONDS * 1000)
)
+
+ /**
+ * Clent side test: validate system env before launch
+ */
@BeforeClass
- public static void checkWindowsSupport() {
- if (Shell.WINDOWS) {
-// assertNotNull("winutils.exe not found", Shell.WINUTILS)
- if (!Shell.WINUTILS) {
- log.error("winutils.exe not found")
- }
- def lib = System.getProperty("java.library.path")
- log.debug("java.library.path = ${lib}")
- }
- }
+ public static void checkClientEnv() {
+ SliderUtils.validateSliderClientEnvironment(null)
+ }
protected String buildClustername(String clustername) {
return clustername ?: createClusterName()
@@ -245,11 +239,16 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
int numLocalDirs,
int numLogDirs,
boolean startHDFS) {
+
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
name = buildClustername(name)
- miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs)
+ miniCluster = new MiniYARNCluster(
+ name,
+ noOfNodeManagers,
+ numLocalDirs,
+ numLogDirs)
miniCluster.init(conf)
miniCluster.start();
if (startHDFS) {
[15/47] git commit: SLIDER-350 use gawk as awk implementation for
windows; add checks for that and xargs on the command line
Posted by st...@apache.org.
SLIDER-350 use gawk as awk implementation for windows; add checks for that and xargs on the command line
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/690793a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/690793a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/690793a4
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 690793a4f8a82a3b7395271803ae9af89eb2cdda
Parents: 2e3c4c8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 14:00:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 14:00:01 2014 +0100
----------------------------------------------------------------------
.../common/tools/TestWindowsSupport.groovy | 38 ++++++++++++++++++--
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
2 files changed, 37 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/690793a4/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 84356ea..d18fe76 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -105,12 +105,33 @@ class TestWindowsSupport extends SliderTestBase {
}
+
+ @Test
+ public void testHasGawkInstalled() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ assert 0 == exec(["gawk", "--version"])
+ }
+
+ @Test
+ public void testHasXargsInstalled() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ assert 0 == exec(["xargs", "--version"])
+ }
+
+
@Test
public void testEmitKillCommand() throws Throwable {
killJavaProcesses("regionserver", 9)
}
@Test
+ public void testHadoopHomeDefined() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ def hadoopHome = Shell.hadoopHome
+ log.info("HADOOP_HOME=$hadoopHome")
+ }
+
+ @Test
public void testHasWinutils() throws Throwable {
assume(Shell.WINDOWS, "not windows")
SliderUtils.verifyWinUtilsValid()
@@ -123,14 +144,27 @@ class TestWindowsSupport extends SliderTestBase {
assert winUtilsPath
File winUtils = new File(winUtilsPath)
log.debug("Winutils is at $winUtils)")
+
+ ForkedProcessService process = exec([winUtilsPath, "systeminfo"])
+ assert 0 == process.exitCode
+ }
+
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+ public ForkedProcessService exec(ArrayList<String> commands) {
ForkedProcessService process;
process = new ForkedProcessService(
- methodName.methodName,
+ methodName.methodName,
[:],
- [winUtilsPath, "systeminfo"]);
+ commands);
process.init(new Configuration());
EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
process.start();
waiter.waitForServiceToStop(5000);
+ process
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/690793a4/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 58a76af..89f1a34 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -755,7 +755,7 @@ class SliderTestUtils extends Assert {
/*
"jps -l | grep "String" | awk "{print $1}" | xargs -n 1 taskkill /PID"
*/
- GString killCommand = "\"jps -l | grep \"${grepString}\" | awk \"{print \$1}\" | xargs -n 1 taskkill /PID\""
+ GString killCommand = "\"jps -l | grep \"${grepString}\" | gawk \"{print \$1}\" | xargs -n 1 taskkill /PID\""
commandString = ["CMD", "/C", killCommand]
}
Process command = commandString.execute()
[06/47] git commit: SLIDER-181. Storm package updates
Posted by st...@apache.org.
SLIDER-181. Storm package updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/201686e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/201686e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/201686e2
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 201686e2ea76d6f7f80554e5b9ad9471fce152be
Parents: e7dd217
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Sep 2 18:43:53 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Sep 2 18:43:53 2014 -0700
----------------------------------------------------------------------
app-packages/storm/README.txt | 12 ++++-----
app-packages/storm/appConfig.json | 15 +++++------
app-packages/storm/configuration/storm-env.xml | 25 ++++++++++++++++++
app-packages/storm/metainfo.xml | 26 +++----------------
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 ------------
...pache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE | 16 ++++++++++++
app-packages/storm/package/scripts/params.py | 15 ++++++-----
app-packages/storm/package/scripts/service.py | 8 ++++++
.../storm/package/scripts/status_params.py | 4 +--
.../storm/package/templates/config.yaml.j2 | 9 -------
.../storm/package/templates/storm_jaas.conf.j2 | 27 +++++++++++++++++---
app-packages/storm/resources.json | 10 +++-----
12 files changed, 101 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 971cf14..d4df85f 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -18,18 +18,18 @@
How to create a Slider app package for Storm?
To create the app package you will need the Storm tarball copied to a specific location.
-Various configurations provided in this sample are customized for apache-storm-0.9.1.2.1.1.0-237.tar.gz.
+Various configurations provided in this sample are customized for apache-storm-0.9.3.2.2.0.0-578.tar.gz.
So if you use a different version you may need to edit a few config values.
Replace the placeholder tarball for Storm.
- cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/
- rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+ cp ~/Downloads/apache-storm-0.9.3.2.2.0.0-578.tar.gz package/files/
+ rm package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
-Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
- zip -r storm_v091.zip .
+Create a zip package at the root of the package (<slider enlistment>/app-packages/storm/)
+ zip -r apache-storm-0.9.3.zip .
Verify the content using
- unzip -l "$@" storm_v091.zip
+ unzip -l "$@" apache-storm-0.9.3.zip
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index b8454f9..b5fe436 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,28 +3,27 @@
"metadata": {
},
"global": {
- "application.def": "storm_v091.zip",
+ "application.def": "apache-storm-0.9.3.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
+ "system_configs": "core-site",
"site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578",
"site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
"site.global.ganglia_server_port": "8668",
- "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
- "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
+ "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
"site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
"site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
"site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
"site.storm-site.transactional.zookeeper.root": "/transactional",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
"site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
@@ -32,7 +31,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.3.2.2.0.0-578/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/configuration/storm-env.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/configuration/storm-env.xml b/app-packages/storm/configuration/storm-env.xml
index f315eb3..091c08d 100644
--- a/app-packages/storm/configuration/storm-env.xml
+++ b/app-packages/storm/configuration/storm-env.xml
@@ -22,6 +22,31 @@
<configuration>
+ <property>
+ <name>kerberos_domain</name>
+ <value></value>
+ <description>The kerberos domain to be used for this Storm cluster</description>
+ </property>
+ <property>
+ <name>storm_client_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm client to be used to communicate with Nimbus and Zookeeper</description>
+ </property>
+ <property>
+ <name>storm_server_principal_name</name>
+ <value></value>
+ <description>The principal name for the Storm server to be used by Nimbus</description>
+ </property>
+ <property>
+ <name>storm_client_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm client</description>
+ </property>
+ <property>
+ <name>storm_server_keytab</name>
+ <value></value>
+ <description>The keytab file path for Storm server</description>
+ </property>
<!-- storm-env.sh -->
<property>
<name>content</name>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 70d369c..43535f4 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -21,7 +21,7 @@
<application>
<name>STORM</name>
<comment>Apache Hadoop Stream processing framework</comment>
- <version>0.9.1.2.1</version>
+ <version>0.9.3</version>
<exportedConfigs>storm-site</exportedConfigs>
<exportGroups>
@@ -29,10 +29,6 @@
<name>QuickLinks</name>
<exports>
<export>
- <name>app.jmx</name>
- <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
- </export>
- <export>
<name>app.monitor</name>
<value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
</export>
@@ -55,7 +51,7 @@
<commandOrders>
<commandOrder>
<command>NIMBUS-START</command>
- <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED
</requires>
</commandOrder>
<commandOrder>
@@ -67,10 +63,6 @@
<requires>NIMBUS-STARTED</requires>
</commandOrder>
<commandOrder>
- <command>STORM_REST_API-START</command>
- <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
- </commandOrder>
- <commandOrder>
<command>STORM_UI_SERVER-START</command>
<requires>NIMBUS-STARTED</requires>
</commandOrder>
@@ -91,18 +83,6 @@
</component>
<component>
- <name>STORM_REST_API</name>
- <category>MASTER</category>
- <autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-app.jmx</appExports>
- <commandScript>
- <script>scripts/rest_api.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
<name>SUPERVISOR</name>
<category>SLAVE</category>
<autoStartOnFailure>true</autoStartOnFailure>
@@ -150,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ <name>files/apache-storm-0.9.3.2.2.0.0-578.tar.gz</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
deleted file mode 100644
index dd934d5..0000000
--- a/app-packages/storm/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
new file mode 100644
index 0000000..dd934d5
--- /dev/null
+++ b/app-packages/storm/package/files/apache-storm-0.9.3.2.2.0.0-578.tar.gz.REPLACE
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace with the actual storm package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index 3e1620b..a70bcb9 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -34,10 +34,6 @@ user_group = config['configurations']['global']['user_group']
java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-rest_api_port = config['configurations']['global']['rest_api_port']
-rest_api_admin_port = config['configurations']['global']['rest_api_admin_port']
-rest_api_conf_file = format("{conf_dir}/config.yaml")
-rest_lib_dir = format("{app_root}/contrib/storm-rest")
storm_bin = format("{app_root}/bin/storm")
storm_env_sh_template = config['configurations']['storm-env']['content']
@@ -53,6 +49,11 @@ security_enabled = ( not is_empty(_authentication) and _authentication == 'kerbe
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
_kerberos_domain = config['configurations']['storm-env']['kerberos_domain']
- _storm_principal_name = config['configurations']['storm-env']['storm_principal_name']
- storm_jaas_principal = _storm_principal_name.replace('_HOST', _hostname_lowercase)
- storm_keytab_path = config['configurations']['storm-env']['storm_keytab']
+ _storm_client_principal_name = config['configurations']['storm-env']['storm_client_principal_name']
+ _storm_server_principal_name = config['configurations']['storm-env']['storm_server_principal_name']
+
+ storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
+ kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 13fcef2..dc3c733 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -52,6 +52,14 @@ def service(
else:
cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ if params.security_enabled:
+ if name == "nimbus":
+ Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"),
+ user=params.storm_user)
+ else:
+ Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"),
+ user=params.storm_user)
+
Execute(cmd,
not_if=no_op_test,
user=params.storm_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/status_params.py b/app-packages/storm/package/scripts/status_params.py
index 5907446..2bf6870 100644
--- a/app-packages/storm/package/scripts/status_params.py
+++ b/app-packages/storm/package/scripts/status_params.py
@@ -28,10 +28,8 @@ pid_supervisor = format("{pid_dir}/supervisor.pid")
pid_drpc = format("{pid_dir}/drpc.pid")
pid_ui = format("{pid_dir}/ui.pid")
pid_logviewer = format("{pid_dir}/logviewer.pid")
-pid_rest_api = format("{pid_dir}/restapi.pid")
pid_files = {"logviewer":pid_logviewer,
"ui": pid_ui,
"nimbus": pid_nimbus,
"supervisor": pid_supervisor,
- "drpc": pid_drpc,
- "rest_api": pid_rest_api}
\ No newline at end of file
+ "drpc": pid_drpc}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/templates/config.yaml.j2
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/templates/config.yaml.j2 b/app-packages/storm/package/templates/config.yaml.j2
index 32d2c99..aa4ec46 100644
--- a/app-packages/storm/package/templates/config.yaml.j2
+++ b/app-packages/storm/package/templates/config.yaml.j2
@@ -16,15 +16,6 @@
nimbusHost: {{nimbus_host}}
nimbusPort: {{nimbus_port}}
-# HTTP-specific options.
-http:
-
- # The port on which the HTTP server listens for service requests.
- port: {{rest_api_port}}
-
- # The port on which the HTTP server listens for administrative requests.
- adminPort: {{rest_api_admin_port}}
-
{% if ganglia_installed %}
enableGanglia: {{ganglia_installed}}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/package/templates/storm_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2
index 4031d22..e68cd57 100644
--- a/app-packages/storm/package/templates/storm_jaas.conf.j2
+++ b/app-packages/storm/package/templates/storm_jaas.conf.j2
@@ -15,12 +15,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+StormServer {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_server_keytab_path}}"
+ storeKey=true
+ useTicketCache=true
+ renewTGT=true
+ principal="{{storm_jaas_server_principal}}";
+};
+StormClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{storm_client_keytab_path}}"
+ storeKey=true
+ useTicketCache=true
+ renewTGT=true
+ serviceName="nimbus"
+ debug=true
+ principal="{{storm_jaas_client_principal}}";
+};
Client {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
- keyTab="{{storm_keytab_path}}"
+ keyTab="{{storm_client_keytab_path}}"
storeKey=true
- useTicketCache=false
+ useTicketCache=true
+ renewTGT=true
serviceName="zookeeper"
- principal="{{storm_jaas_principal}}";
+ principal="{{storm_jaas_client_principal}}";
};
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/201686e2/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index b184a40..017766c 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -11,20 +11,16 @@
"yarn.role.priority": "1",
"yarn.component.instances": "1"
},
- "STORM_REST_API": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1"
- },
"STORM_UI_SERVER": {
- "yarn.role.priority": "3",
+ "yarn.role.priority": "2",
"yarn.component.instances": "1"
},
"DRPC_SERVER": {
- "yarn.role.priority": "4",
+ "yarn.role.priority": "3",
"yarn.component.instances": "1"
},
"SUPERVISOR": {
- "yarn.role.priority": "5",
+ "yarn.role.priority": "4",
"yarn.component.instances": "1"
}
}
[29/47] git commit: Merge branch 'feature/feature/SLIDER-377' into
develop
Posted by st...@apache.org.
Merge branch 'feature/feature/SLIDER-377' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a0a3ae63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a0a3ae63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a0a3ae63
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a0a3ae632fc34be0137ac71b1564ca319c757632
Parents: 88d72dd 4fc0396
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 18:19:08 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 18:19:08 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 6 ++-
.../agent/actions/TestActionVersion.groovy | 5 +--
.../tools/TestClientResourceRegistration.groovy | 3 ++
.../common/tools/TestConfigHelperHDFS.groovy | 3 ++
.../common/tools/TestMiscSliderUtils.groovy | 3 ++
.../common/tools/TestZKIntegration.groovy | 44 ++++++++++----------
.../slider/providers/agent/AgentTestBase.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 27 ++++++------
8 files changed, 52 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
[16/47] git commit: SLIDER-350 use gawk as awk implementation for
windows; add checks for that and xargs on the command line
Posted by st...@apache.org.
SLIDER-350 use gawk as awk implementation for windows; add checks for that and xargs on the command line
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f9caa9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f9caa9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f9caa9b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 8f9caa9ba8ba5a09df794096d0096356535dce38
Parents: 690793a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 14:01:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 14:01:30 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/common/tools/TestWindowsSupport.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f9caa9b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index d18fe76..61ede62 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -155,7 +155,7 @@ class TestWindowsSupport extends SliderTestBase {
* @param commands
* @return
*/
- public ForkedProcessService exec(ArrayList<String> commands) {
+ public ForkedProcessService exec(List<String> commands) {
ForkedProcessService process;
process = new ForkedProcessService(
methodName.methodName,
[20/47] git commit: SLIDER-146 added sample appConfig that enables
kerberos for Accumulo
Posted by st...@apache.org.
SLIDER-146 added sample appConfig that enables kerberos for Accumulo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6320f618
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6320f618
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6320f618
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 6320f6186dec56f68afbbd638f5ddc0e7dc04bff
Parents: 29a40f5
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu Sep 4 07:15:22 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Sep 4 07:15:22 2014 -0700
----------------------------------------------------------------------
.../src/test/resources/appConfig_kerberos.json | 55 ++++++++++++++++++++
1 file changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6320f618/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/appConfig_kerberos.json b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
new file mode 100644
index 0000000..c9684d4
--- /dev/null
+++ b/app-packages/accumulo/src/test/resources/appConfig_kerberos.json
@@ -0,0 +1,55 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": "${app.package.name}.zip",
+ "java_home": "/usr/lib/jvm/java",
+ "site.global.app_user": "accumulo",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+ "site.global.tserver_heapsize": "256m",
+ "site.global.master_heapsize": "128m",
+ "site.global.monitor_heapsize": "64m",
+ "site.global.gc_heapsize": "64m",
+ "site.global.other_heapsize": "128m",
+ "site.global.hadoop_prefix": "/usr/lib/hadoop",
+ "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.global.zookeeper_home": "/usr/lib/zookeeper",
+ "site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
+ "site.global.accumulo_root_password": "NOT_USED",
+ "site.global.user_group": "accumulo",
+ "site.global.security_enabled": "false",
+ "site.global.ssl_cert_dir": "ssl",
+ "site.global.monitor_protocol": "http",
+ "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.general.kerberos.keytab": "/etc/security/keytabs/accumulo.service.keytab",
+ "site.accumulo-site.general.kerberos.principal": "accumulo/_HOST@EXAMPLE.COM",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.trace.user": "root",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
+ },
+ "credentials": {
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
[14/47] git commit: SLIDER-350 add strict test for windows utils
being valid
Posted by st...@apache.org.
SLIDER-350 add strict test for windows utils being valid
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2e3c4c81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2e3c4c81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2e3c4c81
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 2e3c4c81b799cbf6d06e394c2b17f8f161f68734
Parents: bfdec58
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 13:46:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 13:46:59 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 50 +++++++++++++++++++-
.../common/tools/TestWindowsSupport.groovy | 30 ++++++++++--
2 files changed, 76 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2e3c4c81/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index a864878..eb214db 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
@@ -63,6 +64,7 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
@@ -104,8 +106,9 @@ public final class SliderUtils {
public static final String JAVA_SECURITY_KRB5_REALM =
"java.security.krb5.realm";
public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+ public static final String WINUTILS = "WINUTILS.EXE";
+
-
private SliderUtils() {
}
@@ -1499,4 +1502,49 @@ public final class SliderUtils {
return is;
}
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void verifyWinUtilsValid() throws IOException {
+ if (!Shell.WINDOWS) {
+ return;
+ }
+ String winUtilsPath = Shell.getWinUtilsPath();
+ if (winUtilsPath == null) {
+ throw new FileNotFoundException(WINUTILS + " not found on Path : " +
+ System.getenv("Path"));
+ }
+ File winUtils = new File(winUtilsPath);
+ if (!winUtils.isFile()) {
+ throw new FileNotFoundException(WINUTILS
+ + " at " + winUtilsPath
+ + " is not a file");
+
+ }
+ if (winUtils.length() < 0x100) {
+ throw new FileNotFoundException(WINUTILS
+ + " at " + winUtilsPath
+ + " is too short to be an executable");
+ }
+ // now read two bytes and verify the header.
+ FileReader reader = null;
+ try {
+ int[] header = new int[2];
+ reader = new FileReader(winUtilsPath);
+ header[0] = reader.read();
+ header[1] = reader.read();
+ if (header[0] != 'M' || header[1] != 'Z') {
+ throw new FileNotFoundException(WINUTILS
+ + " at " + winUtilsPath
+ + " is not a windows executable file");
+ }
+ } finally {
+ IOUtils.closeStream(reader);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2e3c4c81/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
index 7f5edf3..84356ea 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestWindowsSupport.groovy
@@ -27,14 +27,16 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.util.Shell
import org.apache.slider.providers.agent.AgentUtils
-import org.apache.slider.test.SliderTestUtils
+import org.apache.slider.server.services.workflow.EndOfServiceWaiter
+import org.apache.slider.server.services.workflow.ForkedProcessService
+import org.apache.slider.test.SliderTestBase
import org.junit.Test
import java.util.regex.Pattern
@CompileStatic
@Slf4j
-class TestWindowsSupport extends SliderTestUtils {
+class TestWindowsSupport extends SliderTestBase {
private static final Pattern hasDriveLetterSpecifier =
Pattern.compile("^/?[a-zA-Z]:");
@@ -94,7 +96,6 @@ class TestWindowsSupport extends SliderTestUtils {
@Test
public void testSliderFS() throws Throwable {
assume(Shell.WINDOWS, "not windows")
-
SliderFileSystem sfs = new SliderFileSystem(new Configuration())
try {
def metainfo = AgentUtils.getApplicationMetainfo(sfs, windowsFile)
@@ -107,6 +108,29 @@ class TestWindowsSupport extends SliderTestUtils {
@Test
public void testEmitKillCommand() throws Throwable {
killJavaProcesses("regionserver", 9)
+ }
+
+ @Test
+ public void testHasWinutils() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ SliderUtils.verifyWinUtilsValid()
+ }
+ @Test
+ public void testExecWinutils() throws Throwable {
+ assume(Shell.WINDOWS, "not windows")
+ def winUtilsPath = Shell.winUtilsPath
+ assert winUtilsPath
+ File winUtils = new File(winUtilsPath)
+ log.debug("Winutils is at $winUtils)")
+ ForkedProcessService process;
+ process = new ForkedProcessService(
+ methodName.methodName,
+ [:],
+ [winUtilsPath, "systeminfo"]);
+ process.init(new Configuration());
+ EndOfServiceWaiter waiter = new EndOfServiceWaiter(process);
+ process.start();
+ waiter.waitForServiceToStop(5000);
}
}
[40/47] git commit: Merge branch 'feature/auto_generate_storm_pkg'
into develop
Posted by st...@apache.org.
Merge branch 'feature/auto_generate_storm_pkg' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56cd8667
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56cd8667
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56cd8667
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 56cd8667223a634eb0a806354122cc7210c5a3f8
Parents: 168a7af 4c80820
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 4 21:52:02 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 4 21:52:02 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 8 +++--
app-packages/storm/appConfig.json | 7 ++--
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 38 +++++++++++++++-----
.../agent/TestAgentProviderService.java | 29 +++++++++++++++
5 files changed, 68 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[04/47] git commit: SLIDER-327 AM to move to optimized,
queued Node review and refresh action
Posted by st...@apache.org.
SLIDER-327 AM to move to optimized, queued Node review and refresh action
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/88b665db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/88b665db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/88b665db
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: 88b665db938325bf2c221614f7ffadefe71be632
Parents: 82cf1f0
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 2 21:01:50 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 2 21:01:50 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 9 +--
.../server/appmaster/RoleLaunchService.java | 8 +--
.../server/appmaster/SliderAppMaster.java | 66 ++++++++++++++------
.../server/appmaster/actions/ActionHalt.java | 6 +-
.../appmaster/actions/ActionKillContainer.java | 20 +++++-
.../appmaster/actions/ActionStartContainer.java | 8 ++-
.../appmaster/actions/ActionStopSlider.java | 6 +-
.../server/appmaster/actions/AsyncAction.java | 47 ++++----------
.../actions/ProviderReportedContainerLoss.java | 13 ++--
.../actions/ProviderStartupCompleted.java | 4 --
.../server/appmaster/actions/QueueAccess.java | 5 ++
.../server/appmaster/actions/QueueService.java | 11 +++-
.../actions/RegisterComponentInstance.java | 5 ++
.../actions/ReviewAndFlexApplicationSize.java | 43 +++++++++++++
.../actions/UnregisterComponentInstance.java | 7 ++-
.../server/appmaster/monkey/ChaosEntry.java | 2 +-
.../TestStandaloneAMMonkeyRestart.groovy | 8 +--
.../providers/accumulo/AccumuloTestBase.groovy | 4 +-
.../minicluster/HBaseMiniClusterTestBase.groovy | 1 -
.../flexing/TestClusterFlex1To1.groovy | 2 +-
20 files changed, 176 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f0da136..f6d761e 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2002,12 +2002,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info("Flexing running cluster");
SliderClusterProtocol appMaster = connect(instance);
SliderClusterOperations clusterOps = new SliderClusterOperations(appMaster);
- if (clusterOps.flex(instanceDefinition.getResources())) {
- log.info("Cluster size updated");
- exitCode = EXIT_SUCCESS;
- } else {
- log.info("Requested size is the same as current size: no change");
- }
+ clusterOps.flex(instanceDefinition.getResources());
+ log.info("application instance size updated");
+ exitCode = EXIT_SUCCESS;
} else {
log.info("No running instance to update");
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index e8b6802..37824c8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
@@ -39,9 +38,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
-import java.util.Queue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
/**
* A service for launching containers
@@ -215,9 +214,8 @@ public class RoleLaunchService
instance.roleId = role.id;
instance.environment = envDescription;
actionQueue.put(new ActionStartContainer("starting " + containerRole,
- 0, container,
- containerLauncher.completeContainerLaunch(),
- instance));
+ container, containerLauncher.completeContainerLaunch(), instance, 0,
+ TimeUnit.MILLISECONDS));
} catch (Exception e) {
log.error("Exception thrown while trying to start {}: {}",
containerRole, e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 51c3b93..9ccb382 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -106,6 +106,7 @@ import org.apache.slider.server.appmaster.actions.ActionStopSlider;
import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
import org.apache.slider.server.appmaster.actions.UnregisterComponentInstance;
import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
@@ -160,9 +161,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_AGENT_CONTEXT_ROOT;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.WS_CONTEXT_ROOT;
-
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@@ -1211,11 +1209,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
}
- try {
- reviewRequestAndReleaseNodes();
- } catch (SliderInternalStateException e) {
- log.warn("Exception while flexing nodes", e);
- }
+ reviewRequestAndReleaseNodes("onContainersCompleted");
}
/**
@@ -1223,10 +1217,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* It should be the only way that anything -even the AM itself on startup-
* asks for nodes.
* @param resources the resource tree
- * @return true if the any requests were made
* @throws IOException
*/
- private boolean flexCluster(ConfTree resources)
+ private void flexCluster(ConfTree resources)
throws IOException, SliderInternalStateException, BadConfigException {
appState.updateResourceDefinitions(resources);
@@ -1238,7 +1231,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// ask for more containers if needed
- return reviewRequestAndReleaseNodes();
+ reviewRequestAndReleaseNodes("flexCluster");
}
/**
@@ -1269,13 +1262,47 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/**
* Look at where the current node state is -and whether it should be changed
+ * @param reason
+ */
+ private synchronized void reviewRequestAndReleaseNodes(String reason) {
+ log.debug("reviewRequestAndReleaseNodes({})", reason);
+ queue(new ReviewAndFlexApplicationSize(reason, 0, TimeUnit.SECONDS));
+ }
+
+ /**
+ * Handle the event requesting a review ... look at the queue and decide
+ * whether to act or not
+ * @param action action triggering the event. It may be put
+ * back into the queue
+ * @throws SliderInternalStateException
*/
- private synchronized boolean reviewRequestAndReleaseNodes()
+ public void handleReviewAndFlexApplicationSize(ReviewAndFlexApplicationSize action)
throws SliderInternalStateException {
- log.debug("in reviewRequestAndReleaseNodes()");
+
+ if ( actionQueues.hasQueuedActionWithAttribute(
+ AsyncAction.ATTR_REVIEWS_APP_SIZE | AsyncAction.ATTR_HALTS_APP)) {
+ // this operation isn't needed at all -existing duplicate or shutdown due
+ return;
+ }
+ // if there is an action which changes cluster size, wait
+ if (actionQueues.hasQueuedActionWithAttribute(
+ AsyncAction.ATTR_CHANGES_APP_SIZE)) {
+ // place the action at the back of the queue
+ actionQueues.put(action);
+ }
+
+ executeNodeReview(action.name);
+ }
+
+ /**
+ * Look at where the current node state is -and whether it should be changed
+ */
+ public synchronized void executeNodeReview(String reason)
+ throws SliderInternalStateException {
+
+ log.debug("in executeNodeReview({})", reason);
if (amCompletionFlag.get()) {
log.info("Ignoring node review operation: shutdown in progress");
- return false;
}
try {
List<AbstractRMOperation> allOperations = appState.reviewRequestAndReleaseNodes();
@@ -1283,16 +1310,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
providerRMOperationHandler.execute(allOperations);
//now apply the operations
executeRMOperations(allOperations);
- return !allOperations.isEmpty();
} catch (TriggerClusterTeardownException e) {
//App state has decided that it is time to exit
log.error("Cluster teardown triggered %s", e);
signalAMComplete(e.getExitCode(), e.toString());
- return false;
}
}
+
+
+
/**
* Shutdown operation: release all containers
*/
@@ -1383,8 +1411,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
String payload = request.getClusterSpec();
ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
ConfTree updatedResources = confTreeSerDeser.fromJson(payload);
- boolean flexed = flexCluster(updatedResources);
- return Messages.FlexClusterResponseProto.newBuilder().setResponse(flexed).build();
+ flexCluster(updatedResources);
+ return Messages.FlexClusterResponseProto.newBuilder().setResponse(true).build();
}
@Override //SliderClusterProtocol
@@ -1609,7 +1637,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
executeRMOperations(appState.releaseContainer(containerId));
// ask for more containers if needed
log.info("Container released; triggering review");
- reviewRequestAndReleaseNodes();
+ reviewRequestAndReleaseNodes("Loss of container");
} else {
log.info("Container not in active set - ignoring");
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index c21e249..e2ad559 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -25,7 +25,7 @@ import org.apache.slider.server.appmaster.state.AppState;
import java.util.concurrent.TimeUnit;
/**
- * Exit a JVM halt.
+ * Exit an emergency JVM halt.
* @see ExitUtil#halt(int, String)
*/
public class ActionHalt extends AsyncAction {
@@ -37,7 +37,9 @@ public class ActionHalt extends AsyncAction {
int status,
String text,
long delay, TimeUnit timeUnit) {
- super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
+
+ // do not declare that this action halts the cluster ... keep it a surprise
+ super("Halt", delay, timeUnit);
this.status = status;
this.text = text;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index c1e7e6e..95bf067 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -30,16 +30,34 @@ import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+/**
+ * Kill a specific container
+ */
public class ActionKillContainer extends AsyncAction {
+ /**
+ * container to kill
+ */
private final ContainerId containerId;
+
+ /**
+ * handler for the operation
+ */
private final RMOperationHandler operationHandler;
+
+ /**
+ * Kill a container
+ * @param containerId container to kill
+ * @param delay
+ * @param timeUnit
+ * @param operationHandler
+ */
public ActionKillContainer(
ContainerId containerId,
long delay,
TimeUnit timeUnit,
RMOperationHandler operationHandler) {
- super("kill container", delay, timeUnit);
+ super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
this.operationHandler = operationHandler;
Preconditions.checkArgument(containerId != null);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
index d95dc74..358c844 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -25,6 +25,7 @@ import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
import java.util.Locale;
+import java.util.concurrent.TimeUnit;
/**
* Start a container
@@ -37,15 +38,16 @@ public class ActionStartContainer extends AsyncAction {
private final RoleInstance instance;
public ActionStartContainer(String name,
- long delay,
Container container,
ContainerLaunchContext ctx,
- RoleInstance instance) {
+ RoleInstance instance,
+ long delay, TimeUnit timeUnit) {
super(
String.format(Locale.ENGLISH,
"%s %s: /",
name , container.getId().toString()),
- delay);
+ delay,
+ timeUnit);
this.container = container;
this.ctx = ctx;
this.instance = instance;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
index f084383..64b8e9e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -25,15 +25,11 @@ import org.apache.slider.server.appmaster.state.AppState;
import java.util.concurrent.TimeUnit;
public class ActionStopSlider extends AsyncAction {
- public ActionStopSlider(String message,
- long delay) {
- super(message, delay, ActionAttributes.HALTS_CLUSTER);
- }
public ActionStopSlider(String name,
long delay,
TimeUnit timeUnit) {
- super(name, delay, timeUnit, ActionAttributes.HALTS_CLUSTER);
+ super(name, delay, timeUnit, ATTR_HALTS_APP);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index 996390d..c8db42d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -23,8 +23,6 @@ import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
import java.io.IOException;
-import java.util.Collections;
-import java.util.EnumSet;
import java.util.concurrent.Delayed;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -35,7 +33,7 @@ public abstract class AsyncAction implements Delayed {
public final String name;
private long nanos;
- private final EnumSet<ActionAttributes> attrs;
+ public final int attrs;
private final long sequenceNumber = sequencer.incrementAndGet();
@@ -51,34 +49,18 @@ public abstract class AsyncAction implements Delayed {
protected AsyncAction(String name,
long delay,
TimeUnit timeUnit) {
- this.name = name;
- this.setNanos(convertAndOffset(delay, timeUnit));
- attrs = EnumSet.noneOf(ActionAttributes.class);
+ this(name, delay, timeUnit, 0);
}
protected AsyncAction(String name,
long delay,
TimeUnit timeUnit,
- EnumSet<ActionAttributes> attrs) {
+ int attrs) {
this.name = name;
this.setNanos(convertAndOffset(delay, timeUnit));
this.attrs = attrs;
}
- protected AsyncAction(String name,
- long delay,
- TimeUnit timeUnit,
- ActionAttributes... attributes) {
- this(name, delay, timeUnit);
- Collections.addAll(attrs, attributes);
- }
-
- protected AsyncAction(String name,
- long delayMillis,
- ActionAttributes... attributes) {
- this(name, delayMillis, TimeUnit.MILLISECONDS);
- }
-
protected long convertAndOffset(long delay, TimeUnit timeUnit) {
return now() + TimeUnit.NANOSECONDS.convert(delay, timeUnit);
}
@@ -118,17 +100,18 @@ public abstract class AsyncAction implements Delayed {
return sb.toString();
}
- protected EnumSet<ActionAttributes> getAttrs() {
+ protected int getAttrs() {
return attrs;
}
/**
- * Ask if an action has a specific attribute
+ * Ask if an action has an of the specified bits set.
+ * This is not an equality test.
* @param attr attribute
- * @return true iff the action has the specific attribute
+ * @return true iff the action has any of the bits in the attr arg set
*/
- public boolean hasAttr(ActionAttributes attr) {
- return attrs.contains(attr);
+ public boolean hasAttr(int attr) {
+ return (attrs & attr) != 0;
}
/**
@@ -148,12 +131,8 @@ public abstract class AsyncAction implements Delayed {
public void setNanos(long nanos) {
this.nanos = nanos;
}
-
- public enum ActionAttributes {
- SHRINKS_CLUSTER,
- EXPANDS_CLUSTER,
- HALTS_CLUSTER,
- }
-
-
+
+ public static final int ATTR_CHANGES_APP_SIZE = 1;
+ public static final int ATTR_HALTS_APP = 2;
+ public static final int ATTR_REVIEWS_APP_SIZE = 4;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
index 2aa67bb..41fe494 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
+import java.util.concurrent.TimeUnit;
+
/**
* Report container loss to the AM
* {@link SliderAppMaster#providerLostContainer(ContainerId)}
@@ -31,13 +33,14 @@ public class ProviderReportedContainerLoss extends AsyncAction {
private final ContainerId containerId;
public ProviderReportedContainerLoss(ContainerId containerId) {
- super("lost container " + containerId);
- this.containerId = containerId;
+ this("lost container", 0, TimeUnit.MILLISECONDS, containerId);
}
- public ProviderReportedContainerLoss(
- ContainerId containerId, long delayMillis) {
- super("lost container " + containerId, delayMillis);
+ public ProviderReportedContainerLoss(String name,
+ long delay,
+ TimeUnit timeUnit,
+ ContainerId containerId) {
+ super(name, delay, timeUnit);
this.containerId = containerId;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
index 4577025..957a35f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
@@ -27,10 +27,6 @@ public class ProviderStartupCompleted extends AsyncAction {
super("ProviderStartupCompleted");
}
- public ProviderStartupCompleted(long delayMillis) {
- super("ProviderStartupCompleted", delayMillis);
- }
-
@Override
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
index cffaf5e..0396891 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
@@ -64,4 +64,9 @@ public interface QueueAccess {
* @return true if the action was found and removed.
*/
boolean removeRenewingAction(String name);
+
+ /**
+ * Look in the immediate queue for any actions of a specific attribute
+ */
+ boolean hasQueuedActionWithAttribute(int attr);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 6ad579d..5b24a35 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -19,7 +19,6 @@
package org.apache.slider.server.appmaster.actions;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.slf4j.Logger;
@@ -149,6 +148,16 @@ implements Runnable, QueueAccess {
}
}
}
+
+ @Override
+ public boolean hasQueuedActionWithAttribute(int attr) {
+ for (AsyncAction action : actionQueue) {
+ if (action.hasAttr(attr)) {
+ return true;
+ }
+ }
+ return false;
+ }
/**
* Run until the queue has been told to stop
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
index a8a6fe2..543c1a8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@ -25,6 +25,11 @@ import org.apache.slider.server.appmaster.state.AppState;
import java.util.concurrent.TimeUnit;
+/**
+ * Notify the app master that it should register a component instance
+ * in the registry
+ * {@link SliderAppMaster#registerComponent(ContainerId)}
+ */
public class RegisterComponentInstance extends AsyncAction {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
new file mode 100644
index 0000000..273f599
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ReviewAndFlexApplicationSize.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Tell the AM to execute the full flex review operation
+ */
+public class ReviewAndFlexApplicationSize extends AsyncAction{
+
+ public ReviewAndFlexApplicationSize(String name,
+ long delay,
+ TimeUnit timeUnit) {
+ super(name, delay, timeUnit, ATTR_REVIEWS_APP_SIZE);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ appMaster.handleReviewAndFlexApplicationSize(this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
index 78d9c1c..575fe8f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
@@ -24,12 +24,17 @@ import org.apache.slider.server.appmaster.state.AppState;
import java.util.concurrent.TimeUnit;
+/**
+ * Tell AM to unregister this component instance
+ * {@link SliderAppMaster#unregisterComponent(ContainerId)}
+ */
public class UnregisterComponentInstance extends AsyncAction {
public final ContainerId containerId;
- public UnregisterComponentInstance(ContainerId containerId, long delay,
+ public UnregisterComponentInstance(ContainerId containerId,
+ long delay,
TimeUnit timeUnit) {
super("UnregisterComponentInstance :" + containerId.toString(),
delay, timeUnit);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 5905d2f..2869fe9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -52,7 +52,7 @@ public class ChaosEntry {
Preconditions.checkArgument(target != null, "null target");
Preconditions.checkArgument(probability > 0, "negative probability");
Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
- "probability over 100%");
+ "probability over 100%: "+ probability);
this.name = name;
this.target = target;
this.probability = probability;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
index 5b17cfa..b37b9a6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -20,28 +20,22 @@ package org.apache.slider.agent.standalone
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.SleepJob
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.api.InternalKeys
-import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.ActionAMSuicideArgs
import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
@CompileStatic
@Slf4j
-
class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
-
@Test
public void testStandaloneAMMonkeyRestart() throws Throwable {
describe "Run a Standalone AM with the Chaos monkey set to kill it"
@@ -55,7 +49,7 @@ class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
[
Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "8",
- Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "75000",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "7500",
],
true,
false)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
index 93b2798..3c5606b 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/test/groovy/org/apache/slider/providers/accumulo/AccumuloTestBase.groovy
@@ -228,9 +228,7 @@ public abstract class AccumuloTestBase extends YarnZKMiniClusterTestBase {
//now flex
describe(
"Flexing " + roleMapToString(flexTarget));
- boolean flexed = 0 == sliderClient.flex(clustername,
- flexTarget
- );
+ sliderClient.flex(clustername, flexTarget);
cd = waitForRoleCount(sliderClient, flexTarget,
accumulo_cluster_startup_to_live_time);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 10de9ac..7712a83 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -353,7 +353,6 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
int masterFlexTarget,
int workerFlexTarget,
boolean testHBaseAfter) {
- int flexTarget
describe(
"Flexing masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
boolean flexed;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/88b665db/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
index fdbbce8..c1265de 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex1To1.groovy
@@ -33,7 +33,7 @@ class TestClusterFlex1To1 extends HBaseMiniClusterTestBase {
@Test
public void testClusterFlex1To1() throws Throwable {
- assert !flexHBaseClusterTestRun(
+ assert flexHBaseClusterTestRun(
"",
1,
1,
[13/47] git commit: SLIDER-350 switch to hadoop 3.0.0 for testing
Posted by st...@apache.org.
SLIDER-350 switch to hadoop 3.0.0 for testing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfdec589
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfdec589
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfdec589
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: bfdec5897997511cef405885bd940da01a5189e7
Parents: 38c816e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Sep 4 12:59:28 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Sep 4 12:59:28 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfdec589/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8292073..454918c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,7 +118,7 @@
<!--
core artifacts
-->
- <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
<accumulo.version>1.6.1-SNAPSHOT</accumulo.version>
[42/47] git commit: SLIDER-401 slider-agent tests not running on
windows
Posted by st...@apache.org.
SLIDER-401 slider-agent tests not running on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9f0b985
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9f0b985
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9f0b985
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: a9f0b985d7bab6387a028df3019d467a69a69626
Parents: 2d6331a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 5 13:15:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 5 13:15:38 2014 +0100
----------------------------------------------------------------------
pom.xml | 1 +
slider-agent/pom.xml | 4 +--
slider-agent/src/test/python/python-wrap | 40 ----------------------
slider-agent/src/test/python/python-wrap.sh | 42 ++++++++++++++++++++++++
slider-core/pom.xml | 4 +--
5 files changed, 47 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9efb5c8..d9b3813 100644
--- a/pom.xml
+++ b/pom.xml
@@ -200,6 +200,7 @@
<maven-deploy-plugin.version>2.7</maven-deploy-plugin.version>
<maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
<maven-enforcer-plugin.version>1.0</maven-enforcer-plugin.version>
+ <maven-exec-plugin.version>1.2.1</maven-exec-plugin.version>
<maven-jar-plugin.version>2.3.1</maven-jar-plugin.version>
<maven.javadoc.version>2.8</maven.javadoc.version>
<maven.project.version>2.4</maven.project.version>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-agent/pom.xml
----------------------------------------------------------------------
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index f26cb11..48dd301 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -59,11 +59,11 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<configuration>
- <executable>${project.basedir}/src/test/python/python-wrap</executable>
+ <executable>python</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
<argument>unitTests.py</argument>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-agent/src/test/python/python-wrap
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap
deleted file mode 100755
index 40dc785..0000000
--- a/slider-agent/src/test/python/python-wrap
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2011 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
-
-# reset settings
-unset PYTHON
-
-# checking for preferable python versions
-if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
- PYTHON=/usr/bin/python2.7
-fi
-
-if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
- PYTHON=/usr/bin/python2.6
-fi
-
-# if no preferable python versions found, try to use system one
-if [[ -z "$PYTHON" ]]; then
- PYTHON=/usr/bin/python
-fi
-
-# execute script
-$PYTHON "$@"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-agent/src/test/python/python-wrap.sh
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/python-wrap.sh b/slider-agent/src/test/python/python-wrap.sh
new file mode 100755
index 0000000..8f30d67
--- /dev/null
+++ b/slider-agent/src/test/python/python-wrap.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export PYTHONPATH=/usr/lib/python2.7/site-packages/common_functions:$PYTHONPATH
+
+# reset settings
+unset PYTHON
+
+# checking for preferable python versions
+if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.7
+ export PYTHONPATH=/usr/lib/python2.7/site-packages/common_functions:$PYTHONPATH
+fi
+
+if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.6
+ export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
+fi
+
+# if no preferable python versions found, try to use system one
+if [[ -z "$PYTHON" ]]; then
+ PYTHON=/usr/bin/python
+fi
+
+# execute script
+$PYTHON "$@"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9f0b985/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index cdcea38..326af88 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -538,7 +538,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <version>${maven-exec-plugin.version}</version>
<configuration>
<executable>java</executable>
<arguments>
@@ -562,7 +562,7 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2.1</version>
+ <version>${maven-exec-plugin.version}</version>
<executions>
<execution>
<goals>