You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/27 09:24:08 UTC
[01/22] git commit: SLIDER-440. Add an env.sh file to limit post
install changes to conf dir (fix a Windows regression)
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-158_slider_diagnostic_option 0ffd2d57d -> f8210a7c0
SLIDER-440. Add an env.sh file to limit post install changes to conf dir (fix a Windows regression)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ab149cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ab149cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ab149cb
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 5ab149cbdc73a02d0945a97556fc14e51dab47d2
Parents: 0ffd2d5
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Sep 19 13:38:03 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Sep 19 13:38:03 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ab149cb/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index b290f07..a17e473 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -19,10 +19,11 @@ import sys
import os
import subprocess
import time
+import platform
from threading import Thread
CONF = "conf"
-
+IS_WINDOWS = platform.system() == "Windows"
LIB = "lib"
ENV_KEYS = ["JAVA_HOME", "HADOOP_CONF_DIR"]
@@ -50,17 +51,18 @@ to explain the code here
"""
def executeEnvSh(confDir):
- envCmd = 'source %s/slider-env.sh && env' % confDir
- command = ['bash', '-c', envCmd]
+ if not IS_WINDOWS:
+ envCmd = 'source %s/slider-env.sh && env' % confDir
+ command = ['bash', '-c', envCmd]
- proc = subprocess.Popen(command, stdout = subprocess.PIPE)
+ proc = subprocess.Popen(command, stdout = subprocess.PIPE)
- for line in proc.stdout:
- (key, _, value) = line.strip().partition("=")
- if key in ENV_KEYS:
- os.environ[key] = value
+ for line in proc.stdout:
+ (key, _, value) = line.strip().partition("=")
+ if key in ENV_KEYS:
+ os.environ[key] = value
- proc.communicate()
+ proc.communicate()
def scriptDir():
[20/22] git commit: SLIDER-443 remove default keystore location
SLIDER-449 Consume both streams for openssl (required for windows)
Posted by sm...@apache.org.
SLIDER-443 remove default keystore location
SLIDER-449 Consume both streams for openssl (required for windows)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/732569db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/732569db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/732569db
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 732569db9468ed7ced6fe5ae750130fdcd455d04
Parents: 964e403
Author: Jon Maron <jm...@hortonworks.com>
Authored: Fri Sep 26 17:07:18 2014 -0400
Committer: Jon Maron <jm...@hortonworks.com>
Committed: Fri Sep 26 17:07:18 2014 -0400
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 12 +-
.../services/security/CertificateManager.java | 48 ++++++-
.../providers/slideram/instance/appconf.json | 3 +-
.../agent/TestAgentAMManagementWS.groovy | 131 ++++++++++++-------
.../apache/slider/test/SliderTestUtils.groovy | 3 +-
5 files changed, 141 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index afbb4a8..d57f7dc 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -782,7 +782,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
builder.init(providerName, instanceDefinition);
builder.propagateFilename();
builder.propagatePrincipals();
- builder.setImageDetailsIfAvailable(buildInfo.getImage(), buildInfo.getAppHomeDir());
+ builder.setImageDetailsIfAvailable(buildInfo.getImage(),
+ buildInfo.getAppHomeDir());
builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
@@ -839,7 +840,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw e;
}
try {
- builder.persist(appconfdir, overwrite);
+ persistInstanceDefinition(overwrite, appconfdir, builder);
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on {} : {}", builder, e);
throw new BadClusterStateException("Failed to save " + clustername
@@ -847,6 +848,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ builder.persist(appconfdir, overwrite);
+ }
+
@VisibleForTesting
public static void replaceTokens(ConfTree conf,
String userName, String clusterName) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
index 8f679e1..d200033 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
import java.text.MessageFormat;
@@ -86,6 +87,42 @@ public class CertificateManager {
return certFile.exists();
}
+ class StreamConsumer extends Thread
+ {
+ InputStream is;
+ boolean logOutput;
+
+ StreamConsumer(InputStream is, boolean logOutput)
+ {
+ this.is = is;
+ this.logOutput = logOutput;
+ }
+
+ StreamConsumer(InputStream is)
+ {
+ this(is, false);
+ }
+
+ public void run()
+ {
+ try
+ {
+ InputStreamReader isr = new InputStreamReader(is,
+ Charset.forName("UTF8"));
+ BufferedReader br = new BufferedReader(isr);
+ String line;
+ while ( (line = br.readLine()) != null)
+ if (logOutput) {
+ LOG.info(line);
+ }
+ } catch (IOException e)
+ {
+ LOG.error("Error during processing of process stream", e);
+ }
+ }
+ }
+
+
/**
* Runs os command
*
@@ -98,12 +135,13 @@ public class CertificateManager {
BufferedReader br= null;
try {
process = Runtime.getRuntime().exec(command);
- br = new BufferedReader(new InputStreamReader(
- process.getInputStream(), Charset.forName("UTF8")));
+ StreamConsumer outputConsumer =
+ new StreamConsumer(process.getInputStream(), true);
+ StreamConsumer errorConsumer =
+ new StreamConsumer(process.getErrorStream());
- while ((line = br.readLine()) != null) {
- LOG.info(line);
- }
+ outputConsumer.start();
+ errorConsumer.start();
try {
process.waitFor();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
----------------------------------------------------------------------
diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
index 89095b1..81239a2 100644
--- a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
+++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json
@@ -12,8 +12,7 @@
"components": {
"slider-appmaster" : {
- "jvm.heapsize": "256M",
- "ssl.server.keystore.location": "/tmp/work/security/keystore.p12"
+ "jvm.heapsize": "256M"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index 6720bda..7a39035 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -22,11 +22,19 @@ import com.sun.jersey.api.client.Client
import com.sun.jersey.api.client.WebResource
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.yarn.exceptions.YarnException
import org.apache.slider.api.StatusKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs
+import org.apache.slider.core.build.InstanceBuilder
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.MapOperations
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.launch.LaunchedApplication
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.LockAcquireFailedException
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
import org.apache.slider.server.services.security.CertificateManager
@@ -113,55 +121,86 @@ class TestAgentAMManagementWS extends AgentTestBase {
assert app_def_path.exists()
assert agt_ver_path.exists()
assert agt_conf_path.exists()
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
- roles,
- [
- ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
- ARG_OPTION, APP_DEF, toURIArg(app_def_path),
- ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
- ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
- ],
- true, true,
- true)
- SliderClient sliderClient = launcher.service
- def report = waitForClusterLive(sliderClient)
- def trackingUrl = report.trackingUrl
- log.info("tracking URL is $trackingUrl")
- def agent_url = trackingUrl + AGENT_URI
-
-
- def status = dumpClusterStatus(sliderClient, "agent AM")
- def liveURL = status.getInfo(StatusKeys.INFO_AM_AGENT_OPS_URL)
- if (liveURL) {
- agent_url = liveURL + AGENT_URI
+ try {
+ setSliderClientClassName(TestSliderClient.name)
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ def report = waitForClusterLive(sliderClient)
+ def trackingUrl = report.trackingUrl
+ log.info("tracking URL is $trackingUrl")
+ def agent_url = trackingUrl + AGENT_URI
+
+
+ def status = dumpClusterStatus(sliderClient, "agent AM")
+ def liveURL = status.getInfo(StatusKeys.INFO_AM_AGENT_OPS_URL)
+ if (liveURL) {
+ agent_url = liveURL + AGENT_URI
+ }
+
+ log.info("Agent is $agent_url")
+ log.info("stacks is ${liveURL}stacks")
+ log.info("conf is ${liveURL}conf")
+
+
+ def sleeptime = 10
+ log.info "sleeping for $sleeptime seconds"
+ Thread.sleep(sleeptime * 1000)
+
+
+ String page = fetchWebPageWithoutError(agent_url);
+ log.info(page);
+
+ //WS get
+ Client client = createTestClient();
+
+
+ WebResource webResource = client.resource(agent_url + "test/register");
+ RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON)
+ .post(
+ RegistrationResponse.class,
+ createDummyJSONRegister());
+
+ //TODO: assert failure as actual agent is not started. This test only starts the AM.
+ assert RegistrationStatus.FAILED == response.getResponseStatus();
+ } finally {
+ setSliderClientClassName(SliderClient.name)
}
- log.info("Agent is $agent_url")
- log.info("stacks is ${liveURL}stacks")
- log.info("conf is ${liveURL}conf")
-
-
- def sleeptime = 10
- log.info "sleeping for $sleeptime seconds"
- Thread.sleep(sleeptime * 1000)
-
-
- String page = fetchWebPageWithoutError(agent_url);
- log.info(page);
-
- //WS get
- Client client = createTestClient();
-
-
- WebResource webResource = client.resource(agent_url + "test/register");
- RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON)
- .post(
- RegistrationResponse.class,
- createDummyJSONRegister());
+ }
- //TODO: assert failure as actual agent is not started. This test only starts the AM.
- assert RegistrationStatus.FAILED == response.getResponseStatus();
-
+ static class TestSliderClient extends SliderClient {
+ @Override
+ protected void persistInstanceDefinition(boolean overwrite,
+ Path appconfdir,
+ InstanceBuilder builder)
+ throws IOException, SliderException, LockAcquireFailedException {
+ AggregateConf conf = builder.getInstanceDescription()
+ conf.getAppConfOperations().getComponent("slider-appmaster").put(
+ "ssl.server.keystore.location",
+ "/tmp/work/security/keystore.p12")
+ super.persistInstanceDefinition(overwrite, appconfdir, builder)
+ }
+
+ @Override
+ LaunchedApplication launchApplication(String clustername,
+ Path clusterDirectory,
+ AggregateConf instanceDefinition,
+ boolean debugAM)
+ throws YarnException, IOException {
+ instanceDefinition.getAppConfOperations().getComponent("slider-appmaster").put(
+ "ssl.server.keystore.location",
+ "/tmp/work/security/keystore.p12")
+ return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 9266634..b6d3e17 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -66,6 +66,7 @@ import static Arguments.ARG_OPTION
@Slf4j
@CompileStatic
class SliderTestUtils extends Assert {
+ static String sliderClientClassName = SliderClient.class.name
public static void describe(String s) {
log.info("");
@@ -480,7 +481,7 @@ class SliderTestUtils extends Assert {
Configuration conf,
List args) {
ServiceLauncher<SliderClient> serviceLauncher =
- new ServiceLauncher<SliderClient>(SliderClient.name);
+ new ServiceLauncher<SliderClient>(sliderClientClassName);
log.debug("slider ${SliderUtils.join(args, " ", false)}")
serviceLauncher.launchService(conf,
[19/22] git commit: SLIDER-461 Asymmetry in default arguments
Posted by sm...@apache.org.
SLIDER-461 Asymmetry in default arguments
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/964e4033
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/964e4033
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/964e4033
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 964e4033d5d0df4c6f15e016417af664f0f8ac31
Parents: bb4e7d3
Author: tedyu <yu...@gmail.com>
Authored: Fri Sep 26 09:14:48 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Sep 26 09:14:48 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/appConfig-default.json | 2 +-
app-packages/hbase/appConfig-default.json | 2 +-
app-packages/storm/appConfig-default.json | 2 +-
.../org/apache/slider/providers/agent/AgentProviderService.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/app-packages/hbase-win/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig-default.json b/app-packages/hbase-win/appConfig-default.json
index 5b97e69..95e79ab 100644
--- a/app-packages/hbase-win/appConfig-default.json
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -24,7 +24,7 @@
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
"site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
"site.hbase-site.hbase.regionserver.port": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/app-packages/hbase/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
index 78479db..c7952df 100644
--- a/app-packages/hbase/appConfig-default.json
+++ b/app-packages/hbase/appConfig-default.json
@@ -33,7 +33,7 @@
"site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
"site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
"site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEFAULT_ZK_PATH}",
"site.hbase-site.hbase.regionserver.info.port": "0",
"site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
"site.hbase-site.hbase.regionserver.port": "0",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/app-packages/storm/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig-default.json b/app-packages/storm/appConfig-default.json
index 683b8cc..2b0eea0 100644
--- a/app-packages/storm/appConfig-default.json
+++ b/app-packages/storm/appConfig-default.json
@@ -29,7 +29,7 @@
"site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
"site.storm-site.drpc.invocations.port": "0",
- "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
+ "site.storm-site.storm.zookeeper.root": "${DEFAULT_ZK_PATH}",
"site.storm-site.transactional.zookeeper.port": "null",
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/964e4033/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 647c6a8..755b9d8 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1427,7 +1427,7 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${NN_URI}", nnuri);
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
- tokens.put("${DEF_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
+ tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
[18/22] git commit: SLIDER-450 rename sample template and resources
files for accumulo
Posted by sm...@apache.org.
SLIDER-450 rename sample template and resources files for accumulo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bb4e7d31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bb4e7d31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bb4e7d31
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: bb4e7d31704d8d711ff85288166192f7d41d2648
Parents: 43f9a94
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Fri Sep 26 07:29:34 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Fri Sep 26 07:29:34 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig-default.json | 57 ++++++++++++++++++++
app-packages/accumulo/appConfig.json | 57 --------------------
app-packages/accumulo/pom.xml | 2 +
app-packages/accumulo/resources-default.json | 39 ++++++++++++++
app-packages/accumulo/resources.json | 39 --------------
app-packages/accumulo/src/assembly/accumulo.xml | 4 +-
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +-
7 files changed, 102 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig-default.json b/app-packages/accumulo/appConfig-default.json
new file mode 100644
index 0000000..62050af
--- /dev/null
+++ b/app-packages/accumulo/appConfig-default.json
@@ -0,0 +1,57 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
+ "java_home": "/usr/lib/jvm/java",
+ "site.global.app_user": "${app.user}",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
+ "site.global.tserver_heapsize": "256m",
+ "site.global.master_heapsize": "128m",
+ "site.global.monitor_heapsize": "64m",
+ "site.global.gc_heapsize": "64m",
+ "site.global.other_heapsize": "128m",
+ "site.global.hadoop_prefix": "${hadoop.dir}/hadoop",
+ "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
+ "site.global.zookeeper_home": "${zk.dir}/zookeeper",
+ "site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
+ "site.global.accumulo_root_password": "NOT_USED",
+ "site.global.user_group": "${app.user.group}",
+ "site.global.security_enabled": "false",
+ "site.global.ssl_cert_dir": "ssl",
+ "site.global.monitor_protocol": "http",
+ "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
+ "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
+ "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+ "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+ "site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}",
+ "site.accumulo-site.general.kerberos.principal": "${accumulo.principal}",
+ "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
+ "site.accumulo-site.tserver.memory.maps.max": "80M",
+ "site.accumulo-site.tserver.cache.data.size": "7M",
+ "site.accumulo-site.tserver.cache.index.size": "20M",
+ "site.accumulo-site.trace.user": "root",
+ "site.accumulo-site.tserver.sort.buffer.size": "50M",
+ "site.accumulo-site.tserver.walog.max.size": "40M",
+ "site.accumulo-site.master.port.client": "0",
+ "site.accumulo-site.trace.port.client": "0",
+ "site.accumulo-site.tserver.port.client": "0",
+ "site.accumulo-site.gc.port.client": "0",
+ "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
+ "site.accumulo-site.monitor.port.log4j": "0",
+ "site.accumulo-site.master.replication.coordinator.port": "0",
+ "site.accumulo-site.replication.receipt.service.port": "0",
+ "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/hadoop/.*.jar,\n${hadoop.dir}/hadoop/lib/.*.jar,\n${hadoop.dir}/hadoop-hdfs/.*.jar,\n${hadoop.dir}/hadoop-mapreduce/.*.jar,\n${hadoop.dir}/hadoop-yarn/.*.jar,"
+ },
+ "credentials": {
+ "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
deleted file mode 100644
index 28fb637..0000000
--- a/app-packages/accumulo/appConfig.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
- "java_home": "/usr/lib/jvm/java",
- "site.global.app_user": "${app.user}",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
- "site.global.tserver_heapsize": "256m",
- "site.global.master_heapsize": "128m",
- "site.global.monitor_heapsize": "64m",
- "site.global.gc_heapsize": "64m",
- "site.global.other_heapsize": "128m",
- "site.global.hadoop_prefix": "/usr/lib/hadoop",
- "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
- "site.global.zookeeper_home": "/usr/lib/zookeeper",
- "site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
- "site.global.accumulo_root_password": "NOT_USED",
- "site.global.user_group": "${app.user.group}",
- "site.global.security_enabled": "false",
- "site.global.ssl_cert_dir": "ssl",
- "site.global.monitor_protocol": "http",
- "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
- "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
- "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
- "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
- "site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}",
- "site.accumulo-site.general.kerberos.principal": "${accumulo.principal}",
- "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
- "site.accumulo-site.tserver.memory.maps.max": "80M",
- "site.accumulo-site.tserver.cache.data.size": "7M",
- "site.accumulo-site.tserver.cache.index.size": "20M",
- "site.accumulo-site.trace.user": "root",
- "site.accumulo-site.tserver.sort.buffer.size": "50M",
- "site.accumulo-site.tserver.walog.max.size": "40M",
- "site.accumulo-site.master.port.client": "0",
- "site.accumulo-site.trace.port.client": "0",
- "site.accumulo-site.tserver.port.client": "0",
- "site.accumulo-site.gc.port.client": "0",
- "site.accumulo-site.monitor.port.client": "${ACCUMULO_MONITOR.ALLOCATED_PORT}",
- "site.accumulo-site.monitor.port.log4j": "0",
- "site.accumulo-site.master.replication.coordinator.port": "0",
- "site.accumulo-site.replication.receipt.service.port": "0",
- "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n/usr/lib/hadoop/.*.jar,\n/usr/lib/hadoop/lib/.*.jar,\n/usr/lib/hadoop-hdfs/.*.jar,\n/usr/lib/hadoop-mapreduce/.*.jar,\n/usr/lib/hadoop-yarn/.*.jar,"
- },
- "credentials": {
- "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks": ["root.initial.password", "instance.secret", "trace.token.property.password"]
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 02a0a72..a2014f2 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -39,6 +39,8 @@
<test.app.pkg.dir>${project.build.directory}</test.app.pkg.dir>
<test.app.resources.dir>${project.build.directory}/test-config</test.app.resources.dir>
<!-- these properties are used in the default and the test appConfigs -->
+ <hadoop.dir>/usr/lib</hadoop.dir> <!-- hadoop expected to be found at ${hadoop.dir}/hadoop -->
+ <zk.dir>${hadoop.dir}</zk.dir> <!-- zookeeper expected to be found at ${zk.dir}/zookeeper -->
<app.user>yarn</app.user>
<app.user.group>hadoop</app.user.group>
<accumulo.keytab></accumulo.keytab>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/resources-default.json b/app-packages/accumulo/resources-default.json
new file mode 100644
index 0000000..f9cae60
--- /dev/null
+++ b/app-packages/accumulo/resources-default.json
@@ -0,0 +1,39 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
+ },
+ "components": {
+ "ACCUMULO_MASTER": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "slider-appmaster": {
+ },
+ "ACCUMULO_TSERVER": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "ACCUMULO_MONITOR": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1",
+ "yarn.memory": "128"
+ },
+ "ACCUMULO_GC": {
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "1",
+ "yarn.memory": "128"
+ },
+ "ACCUMULO_TRACER": {
+ "yarn.role.priority": "5",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/resources.json b/app-packages/accumulo/resources.json
deleted file mode 100644
index f9cae60..0000000
--- a/app-packages/accumulo/resources.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "yarn.log.include.patterns": "",
- "yarn.log.exclude.patterns": "",
- "yarn.log.interval": "0"
- },
- "components": {
- "ACCUMULO_MASTER": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "slider-appmaster": {
- },
- "ACCUMULO_TSERVER": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "ACCUMULO_MONITOR": {
- "yarn.role.priority": "3",
- "yarn.component.instances": "1",
- "yarn.memory": "128"
- },
- "ACCUMULO_GC": {
- "yarn.role.priority": "4",
- "yarn.component.instances": "1",
- "yarn.memory": "128"
- },
- "ACCUMULO_TRACER": {
- "yarn.role.priority": "5",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/src/assembly/accumulo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/assembly/accumulo.xml b/app-packages/accumulo/src/assembly/accumulo.xml
index a75d2b1..2a27638 100644
--- a/app-packages/accumulo/src/assembly/accumulo.xml
+++ b/app-packages/accumulo/src/assembly/accumulo.xml
@@ -29,7 +29,7 @@
<files>
<file>
- <source>appConfig.json</source>
+ <source>appConfig-default.json</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
<fileMode>0755</fileMode>
@@ -63,7 +63,7 @@
<exclude>pom.xml</exclude>
<exclude>src/**</exclude>
<exclude>target/**</exclude>
- <exclude>appConfig.json</exclude>
+ <exclude>appConfig-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bb4e7d31/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 475c4e3..dad7601 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -57,7 +57,8 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
Configuration conf = new Configuration()
FileSystem fs = FileSystem.getLocal(conf)
InputStream stream = SliderUtils.getApplicationResourceInputStream(
- fs, new Path(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE), "appConfig.json");
+ fs, new Path(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE),
+ "appConfig-default.json");
assert stream!=null, "Couldn't pull appConfig.json from app pkg"
ConfTreeSerDeser c = new ConfTreeSerDeser()
ConfTree t = c.fromStream(stream)
[05/22] git commit: SLIDER-180. Changes to allow auto-build of hbase
win package
Posted by sm...@apache.org.
SLIDER-180. Changes to allow auto-build of hbase win package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a1086e04
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a1086e04
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a1086e04
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: a1086e04589839f2e074f2aea3f34723090736a2
Parents: 00af5a2
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Sep 21 23:34:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Sep 21 23:34:12 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/README.txt | 29 +++---
app-packages/hbase-win/appConfig.json | 71 +++++++--------
app-packages/hbase-win/metainfo.xml | 4 +-
...se-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME | 17 ----
app-packages/hbase-win/pom.xml | 94 ++++++++++++++++++++
app-packages/hbase-win/src/assembly/hbase.xml | 68 ++++++++++++++
6 files changed, 214 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1086e04/app-packages/hbase-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
index 625bce6..6865168 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -17,18 +17,25 @@
Create Slider App Package for HBase on Windows
-Download the tarball for HBase:
- e.g. path to tarball ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip
+While appConfig.json and resources.json are not required for the package they
+work well as the default configuration for Slider apps. So it is advisable that
+when you create an application package for Slider, include sample/default
+resources.json and appConfig.json for a minimal Yarn cluster.
-Copy the hbase tarball to package/files
- cp ~/Downloads/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip package/files
- rm -rf package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME
+OPTION-I: Use a downloaded hbase tarball fro Windows
-Edit appConfig.json/metainfo.xml and replace all occurances of
-0.98.0.2.1.5.0-2047-hadoop2 with the correct value.
+****** OPTION - I **
+To create the app package you will need the HBase tarball and invoke mvn command
+with appropriate parameters.
-Create a zip package at the root of the package (<slider enlistment>/app-packages/hbase-win/)
- zip -r hbase-win-v098.zip .
+Command:
+mvn clean package -Phbase-app-package-win -Dpkg.version=<version>
+ -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
-Verify the content using
- zip -Tv hbase-win-v098.zip
+Example:
+mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
+ -Dpkg.name=hbase-0.98.5-hadoop2-bin.zip
+ -Dpkg.src=/Users/user1/Downloads
+
+App package can be found in
+ app-packages/hbase/target/slider-hbase-app-win-package-${pkg.version}.zip
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1086e04/app-packages/hbase-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json
index 94f1343..6f07d34 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -1,44 +1,37 @@
{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/HBASE/${app.package.name}.zip",
- "create.default.zookeeper.node": "true",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
+ "create.default.zookeeper.node": "true",
- "site.global.app_user": "hadoop",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
+ "site.global.app_user": "hadoop",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.hbase_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.global.monitor_protocol": "http",
+ "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
+ "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
- "site.global.hbase_instance_name": "instancename",
- "site.global.hbase_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "http",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
- "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
- "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
- "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
- "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
-
- "site.hbase-env.hbase_master_heapsize": "1024m",
- "site.hbase-env.hbase_regionserver_heapsize": "1024m",
-
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "yarn",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1086e04/app-packages/hbase-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml
index 315d279..c585720 100644
--- a/app-packages/hbase-win/metainfo.xml
+++ b/app-packages/hbase-win/metainfo.xml
@@ -25,7 +25,7 @@
1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner.
2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance.
</comment>
- <version>${hbase.version}</version>
+ <version>${pkg.version}</version>
<type>YARN-APP</type>
<minHadoopVersion>2.1.0</minHadoopVersion>
<exportedConfigs>hbase-site</exportedConfigs>
@@ -96,7 +96,7 @@
<packages>
<package>
<type>zip</type>
- <name>files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip</name>
+ <name>files/${pkg.name}</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1086e04/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME b/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME
deleted file mode 100644
index 96ed9f1..0000000
--- a/app-packages/hbase-win/package/files/hbase-0.98.0.2.1.5.0-2047-hadoop2.zip.REPLACEME
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with the actual HBase package for Windows.
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1086e04/app-packages/hbase-win/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/pom.xml b/app-packages/hbase-win/pom.xml
new file mode 100644
index 0000000..ad5855e
--- /dev/null
+++ b/app-packages/hbase-win/pom.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-hbase-app-win-package</artifactId>
+ <packaging>pom</packaging>
+ <name>Slider HBase App Package for Windows</name>
+ <description>Slider HBase App Package for Windows</description>
+ <version>${bld.version}</version>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ <app.package.name>${project.artifactId}-${project.version}</app.package.name>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>hbase-app-package-win</id>
+ <properties>
+ <bld.version>${pkg.version}</bld.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}"/>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptor>src/assembly/hbase.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ </configuration>
+ <executions>
+ <execution>
+ <id>build-tarball</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+ <build>
+ </build>
+
+ <dependencies>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1086e04/app-packages/hbase-win/src/assembly/hbase.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/src/assembly/hbase.xml b/app-packages/hbase-win/src/assembly/hbase.xml
new file mode 100644
index 0000000..e62da84
--- /dev/null
+++ b/app-packages/hbase-win/src/assembly/hbase.xml
@@ -0,0 +1,68 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hbase_v${hbase.version}</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <files>
+ <file>
+ <source>appConfig.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>metainfo.xml</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
+ <source>${pkg.src}/${pkg.name}</source>
+ <outputDirectory>package/files</outputDirectory>
+ <filtered>false</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ </files>
+
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <outputDirectory>/</outputDirectory>
+ <excludes>
+ <exclude>pom.xml</exclude>
+ <exclude>src/**</exclude>
+ <exclude>target/**</exclude>
+ <exclude>appConfig.json</exclude>
+ <exclude>metainfo.xml</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <directoryMode>0755</directoryMode>
+ </fileSet>
+
+ </fileSets>
+</assembly>
[04/22] git commit: SLIDER-434 fix accumulo app package funtests not
to reference agent tarball
Posted by sm...@apache.org.
SLIDER-434 fix accumulo app package funtests not to reference agent tarball
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/00af5a2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/00af5a2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/00af5a2a
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 00af5a2a694a3cd115a818e046a223020c4163dc
Parents: e3c49d5
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Sun Sep 21 19:30:11 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Sun Sep 21 19:30:11 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/00af5a2a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index f09d3ef..475c4e3 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -129,7 +129,6 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, getClusterName(),
- ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
[21/22] git commit: SLIDER-389. Upgrade kazoo code base to the latest
version
Posted by sm...@apache.org.
SLIDER-389. Upgrade kazoo code base to the latest version
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ce647d39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ce647d39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ce647d39
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: ce647d399acf5b2d90e51a8509633183438e11d3
Parents: 732569d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Sep 26 14:20:49 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Sep 26 14:20:49 2014 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/kazoo/client.py | 56 +++++++-----
.../src/main/python/kazoo/handlers/utils.py | 42 +--------
.../main/python/kazoo/protocol/connection.py | 74 ++++++++--------
.../src/main/python/kazoo/tests/test_client.py | 55 ++++++------
.../main/python/kazoo/tests/test_connection.py | 89 ++++++++------------
5 files changed, 137 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce647d39/slider-agent/src/main/python/kazoo/client.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/client.py b/slider-agent/src/main/python/kazoo/client.py
index 11d9008..47545ee 100644
--- a/slider-agent/src/main/python/kazoo/client.py
+++ b/slider-agent/src/main/python/kazoo/client.py
@@ -20,7 +20,7 @@ from kazoo.exceptions import (
WriterNotClosedException,
)
from kazoo.handlers.threading import SequentialThreadingHandler
-from kazoo.handlers.utils import capture_exceptions, wrap, pipe_or_sock_write
+from kazoo.handlers.utils import capture_exceptions, wrap
from kazoo.hosts import collect_hosts
from kazoo.loggingsupport import BLATHER
from kazoo.protocol.connection import ConnectionHandler
@@ -220,7 +220,6 @@ class KazooClient(object):
elif type(command_retry) is KazooRetry:
self.retry = command_retry
-
if type(self._conn_retry) is KazooRetry:
if self.handler.sleep_func != self._conn_retry.sleep_func:
raise ConfigurationError("Retry handler and event handler "
@@ -228,19 +227,21 @@ class KazooClient(object):
if type(self.retry) is KazooRetry:
if self.handler.sleep_func != self.retry.sleep_func:
- raise ConfigurationError("Command retry handler and event "
- "handler must use the same sleep func")
+ raise ConfigurationError(
+ "Command retry handler and event handler "
+ "must use the same sleep func")
if self.retry is None or self._conn_retry is None:
old_retry_keys = dict(_RETRY_COMPAT_DEFAULTS)
for key in old_retry_keys:
try:
old_retry_keys[key] = kwargs.pop(key)
- warnings.warn('Passing retry configuration param %s to the'
- ' client directly is deprecated, please pass a'
- ' configured retry object (using param %s)' % (
- key, _RETRY_COMPAT_MAPPING[key]),
- DeprecationWarning, stacklevel=2)
+ warnings.warn(
+ 'Passing retry configuration param %s to the '
+ 'client directly is deprecated, please pass a '
+ 'configured retry object (using param %s)' % (
+ key, _RETRY_COMPAT_MAPPING[key]),
+ DeprecationWarning, stacklevel=2)
except KeyError:
pass
@@ -258,12 +259,13 @@ class KazooClient(object):
**retry_keys)
self._conn_retry.interrupt = lambda: self._stopped.is_set()
- self._connection = ConnectionHandler(self, self._conn_retry.copy(),
- logger=self.logger)
+ self._connection = ConnectionHandler(
+ self, self._conn_retry.copy(), logger=self.logger)
# Every retry call should have its own copy of the retry helper
# to avoid shared retry counts
self._retry = self.retry
+
def _retry(*args, **kwargs):
return self._retry.copy()(*args, **kwargs)
self.retry = _retry
@@ -282,7 +284,7 @@ class KazooClient(object):
self.Semaphore = partial(Semaphore, self)
self.ShallowParty = partial(ShallowParty, self)
- # If we got any unhandled keywords, complain like python would
+ # If we got any unhandled keywords, complain like Python would
if kwargs:
raise TypeError('__init__() got unexpected keyword arguments: %s'
% (kwargs.keys(),))
@@ -433,7 +435,8 @@ class KazooClient(object):
return
if state in (KeeperState.CONNECTED, KeeperState.CONNECTED_RO):
- self.logger.info("Zookeeper connection established, state: %s", state)
+ self.logger.info("Zookeeper connection established, "
+ "state: %s", state)
self._live.set()
self._make_state_change(KazooState.CONNECTED)
elif state in LOST_STATES:
@@ -510,12 +513,12 @@ class KazooClient(object):
self._queue.append((request, async_object))
# wake the connection, guarding against a race with close()
- write_pipe = self._connection._write_pipe
- if write_pipe is None:
+ write_sock = self._connection._write_sock
+ if write_sock is None:
async_object.set_exception(ConnectionClosedError(
"Connection has been closed"))
try:
- pipe_or_sock_write(write_pipe, b'\0')
+ write_sock.send(b'\0')
except:
async_object.set_exception(ConnectionClosedError(
"Connection has been closed"))
@@ -585,7 +588,7 @@ class KazooClient(object):
self._stopped.set()
self._queue.append((CloseInstance, None))
- pipe_or_sock_write(self._connection._write_pipe, b'\0')
+ self._connection._write_sock.send(b'\0')
self._safe_close()
def restart(self):
@@ -622,7 +625,7 @@ class KazooClient(object):
if not self._live.is_set():
raise ConnectionLoss("No connection to server")
- peer = self._connection._socket.getpeername()
+ peer = self._connection._socket.getpeername()[:2]
sock = self.handler.create_connection(
peer, timeout=self._session_timeout / 1000.0)
sock.sendall(cmd)
@@ -786,7 +789,7 @@ class KazooClient(object):
"""
acl = acl or self.default_acl
return self.create_async(path, value, acl=acl, ephemeral=ephemeral,
- sequence=sequence, makepath=makepath).get()
+ sequence=sequence, makepath=makepath).get()
def create_async(self, path, value=b"", acl=None, ephemeral=False,
sequence=False, makepath=False):
@@ -828,7 +831,8 @@ class KazooClient(object):
@capture_exceptions(async_result)
def do_create():
- result = self._create_async_inner(path, value, acl, flags, trailing=sequence)
+ result = self._create_async_inner(
+ path, value, acl, flags, trailing=sequence)
result.rawlink(create_completion)
@capture_exceptions(async_result)
@@ -867,10 +871,13 @@ class KazooClient(object):
return async_result
def ensure_path(self, path, acl=None):
- """Recursively create a path if it doesn't exist.
+ """Recursively create a path if it doesn't exist. Also return value indicates
+ if path already existed or had to be created.
:param path: Path of node.
:param acl: Permissions for node.
+ :returns `True` if path existed, `False` otherwise.
+ :rtype: bool
"""
return self.ensure_path_async(path, acl).get()
@@ -1291,6 +1298,13 @@ class TransactionRequest(object):
Transactions are not thread-safe and should not be accessed from
multiple threads at once.
+ .. note::
+
+ The ``committed`` attribute only indicates whether this
+ transaction has been sent to Zookeeper and is used to prevent
+ duplicate commits of the same transaction. The result should be
+ checked to determine if the transaction executed as desired.
+
.. versionadded:: 0.6
Requires Zookeeper 3.4+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce647d39/slider-agent/src/main/python/kazoo/handlers/utils.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/utils.py b/slider-agent/src/main/python/kazoo/handlers/utils.py
index 385495e..93cfdb5 100644
--- a/slider-agent/src/main/python/kazoo/handlers/utils.py
+++ b/slider-agent/src/main/python/kazoo/handlers/utils.py
@@ -23,47 +23,7 @@ def _set_default_tcpsock_options(module, sock):
_set_fd_cloexec(sock)
return sock
-def pipe_or_sock_read(p_or_s, n):
- ''' Use a socket or a pipe to read something'''
- if isinstance(p_or_s, int):
- # This is a pipe
- return os.read(p_or_s, n)
- else:
- return p_or_s.recv(n)
-
-def pipe_or_sock_close(p_or_s):
- ''' Closes either a socket or a pipe'''
- if isinstance(p_or_s, int):
- os.close(p_or_s)
- else:
- p_or_s.close()
-
-def pipe_or_sock_write(p_or_s, b):
- ''' Read from a socket or a pipe depending on what is passed'''
- if isinstance(p_or_s, int):
- # This is a pipe
- os.write(p_or_s,b)
- else:
- p_or_s.send(b)
-
-def create_pipe_or_sock():
- """ Create a non-blocking read/write pipe.
- On Windows create a pair of sockets
- """
- if sys.platform == "win32":
- r, w = create_sock_pair()
- else:
- r, w = os.pipe()
- if HAS_FNCTL:
- fcntl.fcntl(r, fcntl.F_SETFL, os.O_NONBLOCK)
- fcntl.fcntl(w, fcntl.F_SETFL, os.O_NONBLOCK)
- _set_fd_cloexec(r)
- _set_fd_cloexec(w)
- return r, w
-
-
-
-def create_sock_pair(port=0):
+def create_socket_pair(port=0):
"""Create socket pair.
If socket.socketpair isn't available, we emulate it.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce647d39/slider-agent/src/main/python/kazoo/protocol/connection.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/connection.py b/slider-agent/src/main/python/kazoo/protocol/connection.py
index 7872928..6b89c18 100644
--- a/slider-agent/src/main/python/kazoo/protocol/connection.py
+++ b/slider-agent/src/main/python/kazoo/protocol/connection.py
@@ -17,7 +17,7 @@ from kazoo.exceptions import (
SessionExpiredError,
NoNodeError
)
-from kazoo.handlers.utils import create_pipe_or_sock, pipe_or_sock_read, pipe_or_sock_write, pipe_or_sock_close
+from kazoo.handlers.utils import create_socket_pair
from kazoo.loggingsupport import BLATHER
from kazoo.protocol.serialization import (
Auth,
@@ -146,8 +146,8 @@ class ConnectionHandler(object):
self.connection_stopped.set()
self.ping_outstanding = client.handler.event_object()
- self._read_pipe = None
- self._write_pipe = None
+ self._read_sock = None
+ self._write_sock = None
self._socket = None
self._xid = None
@@ -169,7 +169,7 @@ class ConnectionHandler(object):
def start(self):
"""Start the connection up"""
if self.connection_closed.is_set():
- self._read_pipe, self._write_pipe = create_pipe_or_sock()
+ self._read_sock, self._write_sock = create_socket_pair()
self.connection_closed.clear()
if self._connection_routine:
raise Exception("Unable to start, connection routine already "
@@ -192,12 +192,12 @@ class ConnectionHandler(object):
if not self.connection_stopped.is_set():
raise Exception("Cannot close connection until it is stopped")
self.connection_closed.set()
- wp, rp = self._write_pipe, self._read_pipe
- self._write_pipe = self._read_pipe = None
- if wp is not None:
- pipe_or_sock_close(wp)
- if rp is not None:
- pipe_or_sock_close(rp)
+ ws, rs = self._write_sock, self._read_sock
+ self._write_sock = self._read_sock = None
+ if ws is not None:
+ ws.close()
+ if rs is not None:
+ rs.close()
def _server_pinger(self):
"""Returns a server pinger iterable, that will ping the next
@@ -238,8 +238,8 @@ class ConnectionHandler(object):
if xid:
header, buffer, offset = self._read_header(timeout)
if header.xid != xid:
- raise RuntimeError('xids do not match, expected %r received %r',
- xid, header.xid)
+ raise RuntimeError('xids do not match, expected %r '
+ 'received %r', xid, header.xid)
if header.zxid > 0:
zxid = header.zxid
if header.err:
@@ -257,8 +257,9 @@ class ConnectionHandler(object):
try:
obj, _ = request.deserialize(msg, 0)
except Exception:
- self.logger.exception("Exception raised during deserialization"
- " of request: %s", request)
+ self.logger.exception(
+ "Exception raised during deserialization "
+ "of request: %s", request)
# raise ConnectionDropped so connect loop will retry
raise ConnectionDropped('invalid server response')
@@ -276,8 +277,9 @@ class ConnectionHandler(object):
if request.type:
b.extend(int_struct.pack(request.type))
b += request.serialize()
- self.logger.log((BLATHER if isinstance(request, Ping) else logging.DEBUG),
- "Sending request(xid=%s): %s", xid, request)
+ self.logger.log(
+ (BLATHER if isinstance(request, Ping) else logging.DEBUG),
+ "Sending request(xid=%s): %s", xid, request)
self._write(int_struct.pack(len(b)) + b, timeout)
def _write(self, msg, timeout):
@@ -358,8 +360,9 @@ class ConnectionHandler(object):
try:
response = request.deserialize(buffer, offset)
except Exception as exc:
- self.logger.exception("Exception raised during deserialization"
- " of request: %s", request)
+ self.logger.exception(
+ "Exception raised during deserialization "
+ "of request: %s", request)
async_object.set_exception(exc)
return
self.logger.debug(
@@ -415,11 +418,11 @@ class ConnectionHandler(object):
except IndexError:
# Not actually something on the queue, this can occur if
# something happens to cancel the request such that we
- # don't clear the pipe below after sending
+ # don't clear the socket below after sending
try:
# Clear possible inconsistence (no request in the queue
- # but have data in the read pipe), which causes cpu to spin.
- pipe_or_sock_read(self._read_pipe, 1)
+ # but have data in the read socket), which causes cpu to spin.
+ self._read_sock.recv(1)
except OSError:
pass
return
@@ -440,7 +443,7 @@ class ConnectionHandler(object):
self._submit(request, connect_timeout, xid)
client._queue.popleft()
- pipe_or_sock_read(self._read_pipe, 1)
+ self._read_sock.recv(1)
client._pending.append((request, async_object, xid))
def _send_ping(self, connect_timeout):
@@ -519,7 +522,7 @@ class ConnectionHandler(object):
jitter_time = random.randint(0, 40) / 100.0
# Ensure our timeout is positive
timeout = max([read_timeout / 2.0 - jitter_time, jitter_time])
- s = self.handler.select([self._socket, self._read_pipe],
+ s = self.handler.select([self._socket, self._read_sock],
[], [], timeout)[0]
if not s:
@@ -570,9 +573,9 @@ class ConnectionHandler(object):
self.logger.info('Connecting to %s:%s', host, port)
self.logger.log(BLATHER,
- ' Using session_id: %r session_passwd: %s',
- client._session_id,
- hexlify(client._session_passwd))
+ ' Using session_id: %r session_passwd: %s',
+ client._session_id,
+ hexlify(client._session_passwd))
with self._socket_error_handling():
self._socket = self.handler.create_connection(
@@ -584,7 +587,8 @@ class ConnectionHandler(object):
client._session_id or 0, client._session_passwd,
client.read_only)
- connect_result, zxid = self._invoke(client._session_timeout, connect)
+ connect_result, zxid = self._invoke(
+ client._session_timeout / 1000.0, connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
@@ -601,13 +605,13 @@ class ConnectionHandler(object):
client._session_passwd = connect_result.passwd
self.logger.log(BLATHER,
- 'Session created, session_id: %r session_passwd: %s\n'
- ' negotiated session timeout: %s\n'
- ' connect timeout: %s\n'
- ' read timeout: %s', client._session_id,
- hexlify(client._session_passwd),
- negotiated_session_timeout, connect_timeout,
- read_timeout)
+ 'Session created, session_id: %r session_passwd: %s\n'
+ ' negotiated session timeout: %s\n'
+ ' connect timeout: %s\n'
+ ' read timeout: %s', client._session_id,
+ hexlify(client._session_passwd),
+ negotiated_session_timeout, connect_timeout,
+ read_timeout)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
@@ -618,7 +622,7 @@ class ConnectionHandler(object):
for scheme, auth in client.auth_data:
ap = Auth(0, scheme, auth)
- zxid = self._invoke(connect_timeout, ap, xid=AUTH_XID)
+ zxid = self._invoke(connect_timeout / 1000.0, ap, xid=AUTH_XID)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce647d39/slider-agent/src/main/python/kazoo/tests/test_client.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_client.py b/slider-agent/src/main/python/kazoo/tests/test_client.py
index 5807292..f851b63 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_client.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_client.py
@@ -72,7 +72,8 @@ class TestClientConstructor(unittest.TestCase):
def test_invalid_handler(self):
from kazoo.handlers.threading import SequentialThreadingHandler
- self.assertRaises(ConfigurationError,
+ self.assertRaises(
+ ConfigurationError,
self._makeOne, handler=SequentialThreadingHandler)
def test_chroot(self):
@@ -91,7 +92,7 @@ class TestClientConstructor(unittest.TestCase):
def test_ordered_host_selection(self):
client = self._makeOne(hosts='127.0.0.1:9,127.0.0.2:9/a',
- randomize_hosts=False)
+ randomize_hosts=False)
hosts = [h for h in client.hosts]
eq_(hosts, [('127.0.0.1', 9), ('127.0.0.2', 9)])
@@ -371,29 +372,29 @@ class TestConnection(KazooTestCase):
client = self.client
client.stop()
- write_pipe = client._connection._write_pipe
+ write_sock = client._connection._write_sock
- # close the connection to free the pipe
+ # close the connection to free the socket
client.close()
- eq_(client._connection._write_pipe, None)
+ eq_(client._connection._write_sock, None)
# sneak in and patch client to simulate race between a thread
# calling stop(); close() and one running a command
oldstate = client._state
client._state = KeeperState.CONNECTED
- client._connection._write_pipe = write_pipe
+ client._connection._write_sock = write_sock
try:
- # simulate call made after write pipe is closed
+ # simulate call made after write socket is closed
self.assertRaises(ConnectionClosedError, client.exists, '/')
- # simualte call made after write pipe is set to None
- client._connection._write_pipe = None
+ # simulate call made after write socket is set to None
+ client._connection._write_sock = None
self.assertRaises(ConnectionClosedError, client.exists, '/')
finally:
# reset for teardown
client._state = oldstate
- client._connection._write_pipe = None
+ client._connection._write_sock = None
class TestClient(KazooTestCase):
@@ -544,9 +545,9 @@ class TestClient(KazooTestCase):
client = self.client
client.create("/1", b"ephemeral", ephemeral=True)
self.assertRaises(NoChildrenForEphemeralsError,
- client.create, "/1/2", b"val1")
+ client.create, "/1/2", b"val1")
self.assertRaises(NoChildrenForEphemeralsError,
- client.create, "/1/2", b"val1", ephemeral=True)
+ client.create, "/1/2", b"val1", ephemeral=True)
def test_create_sequence(self):
client = self.client
@@ -560,8 +561,8 @@ class TestClient(KazooTestCase):
def test_create_ephemeral_sequence(self):
basepath = "/" + uuid.uuid4().hex
- realpath = self.client.create(basepath, b"sandwich", sequence=True,
- ephemeral=True)
+ realpath = self.client.create(basepath, b"sandwich",
+ sequence=True, ephemeral=True)
self.assertTrue(basepath != realpath and realpath.startswith(basepath))
data, stat = self.client.get(realpath)
eq_(data, b"sandwich")
@@ -575,33 +576,35 @@ class TestClient(KazooTestCase):
data, stat = self.client.get("/1/2/3/4/5")
eq_(data, b"val2")
- self.assertRaises(NodeExistsError, self.client.create, "/1/2/3/4/5",
- b"val2", makepath=True)
+ self.assertRaises(NodeExistsError, self.client.create,
+ "/1/2/3/4/5", b"val2", makepath=True)
def test_create_makepath_incompatible_acls(self):
from kazoo.client import KazooClient
from kazoo.security import make_digest_acl_credential, CREATOR_ALL_ACL
credential = make_digest_acl_credential("username", "password")
- alt_client = KazooClient(self.cluster[0].address + self.client.chroot,
+ alt_client = KazooClient(
+ self.cluster[0].address + self.client.chroot,
max_retries=5, auth_data=[("digest", credential)])
alt_client.start()
alt_client.create("/1/2", b"val2", makepath=True, acl=CREATOR_ALL_ACL)
try:
- self.assertRaises(NoAuthError, self.client.create, "/1/2/3/4/5",
- b"val2", makepath=True)
+ self.assertRaises(NoAuthError, self.client.create,
+ "/1/2/3/4/5", b"val2", makepath=True)
finally:
alt_client.delete('/', recursive=True)
alt_client.stop()
def test_create_no_makepath(self):
- self.assertRaises(NoNodeError, self.client.create, "/1/2", b"val1")
- self.assertRaises(NoNodeError, self.client.create, "/1/2", b"val1",
- makepath=False)
+ self.assertRaises(NoNodeError, self.client.create,
+ "/1/2", b"val1")
+ self.assertRaises(NoNodeError, self.client.create,
+ "/1/2", b"val1", makepath=False)
self.client.create("/1/2", b"val1", makepath=True)
- self.assertRaises(NoNodeError, self.client.create, "/1/2/3/4", b"val1",
- makepath=False)
+ self.assertRaises(NoNodeError, self.client.create,
+ "/1/2/3/4", b"val1", makepath=False)
def test_create_exists(self):
from kazoo.exceptions import NodeExistsError
@@ -844,7 +847,7 @@ class TestClient(KazooTestCase):
client = self.client
self.assertRaises(NoNodeError, client.get_children, '/none')
self.assertRaises(NoNodeError, client.get_children,
- '/none', include_data=True)
+ '/none', include_data=True)
def test_get_children_invalid_path(self):
client = self.client
@@ -855,7 +858,7 @@ class TestClient(KazooTestCase):
self.assertRaises(TypeError, client.get_children, ('a', 'b'))
self.assertRaises(TypeError, client.get_children, 'a', watch=True)
self.assertRaises(TypeError, client.get_children,
- 'a', include_data='yes')
+ 'a', include_data='yes')
def test_invalid_auth(self):
from kazoo.exceptions import AuthFailedError
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ce647d39/slider-agent/src/main/python/kazoo/tests/test_connection.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_connection.py b/slider-agent/src/main/python/kazoo/tests/test_connection.py
index d89a2c6..c8c4581 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_connection.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_connection.py
@@ -43,8 +43,9 @@ class Delete(namedtuple('Delete', 'path version')):
class TestConnectionHandler(KazooTestCase):
def test_bad_deserialization(self):
async_object = self.client.handler.async_result()
- self.client._queue.append((Delete(self.client.chroot, -1), async_object))
- os.write(self.client._connection._write_pipe, b'\0')
+ self.client._queue.append(
+ (Delete(self.client.chroot, -1), async_object))
+ self.client._connection._write_sock.send(b'\0')
@raises(ValueError)
def testit():
@@ -185,75 +186,51 @@ class TestConnectionHandler(KazooTestCase):
# should be able to restart
self.client.start()
- def test_connection_pipe(self):
+ def test_connection_sock(self):
client = self.client
- read_pipe = client._connection._read_pipe
- write_pipe = client._connection._write_pipe
+ read_sock = client._connection._read_sock
+ write_sock = client._connection._write_sock
- assert read_pipe is not None
- assert write_pipe is not None
+ assert read_sock is not None
+ assert write_sock is not None
- # stop client and pipe should not yet be closed
+ # stop client and socket should not yet be closed
client.stop()
- assert read_pipe is not None
- assert write_pipe is not None
- if sys.platform != "win32"
- os.fstat(read_pipe)
- os.fstat(write_pipe)
- else:
- read_pipe.getsockname()
- write_pipe.getsockname()
+ assert read_sock is not None
+ assert write_sock is not None
+
+ read_sock.getsockname()
+ write_sock.getsockname()
- # close client, and pipes should be
+ # close client, and sockets should be closed
client.close()
- if sys.platform != "win32"
- try:
- os.fstat(read_pipe)
- except OSError as e:
- if not e.errno == errno.EBADF:
- raise
- else:
- self.fail("Expected read_pipe to be closed")
-
- try:
- os.fstat(write_pipe)
- except OSError as e:
- if not e.errno == errno.EBADF:
- raise
- else:
- self.fail("Expected write_pipe to be closed")
- else:
- pass # Not sure what to do here
-
- # start client back up. should get a new, valid pipe
+ # Todo check socket closing
+
+ # start client back up. should get a new, valid socket
client.start()
- read_pipe = client._connection._read_pipe
- write_pipe = client._connection._write_pipe
-
- assert read_pipe is not None
- assert write_pipe is not None
- if sys.platform != "win32"
- os.fstat(read_pipe)
- os.fstat(write_pipe)
- else:
- read_pipe.getsockname()
- write_pipe.getsockname()
+ read_sock = client._connection._read_sock
+ write_sock = client._connection._write_sock
+
+ assert read_sock is not None
+ assert write_sock is not None
+ read_sock.getsockname()
+ write_sock.getsockname()
- def test_dirty_pipe(self):
+ def test_dirty_sock(self):
client = self.client
- read_pipe = client._connection._read_pipe
- write_pipe = client._connection._write_pipe
+ read_sock = client._connection._read_sock
+ write_sock = client._connection._write_sock
- # add a stray byte to the pipe and ensure that doesn't
+ # add a stray byte to the socket and ensure that doesn't
# blow up client. simulates case where some error leaves
- # a byte in the pipe which doesn't correspond to the
+ # a byte in the socket which doesn't correspond to the
# request queue.
- pipe_or_sock_write(write_pipe, b'\0')
+ write_sock.send(b'\0')
- # eventually this byte should disappear from pipe
- wait(lambda: client.handler.select([read_pipe], [], [], 0)[0] == [])
+ # eventually this byte should disappear from socket
+ wait(lambda: client.handler.select([read_sock], [], [], 0)[0] == [])
class TestConnectionDrop(KazooTestCase):
[17/22] git commit: SLIDER-459. Environment variable for containers
do not expand correctly
Posted by sm...@apache.org.
SLIDER-459. Environment variable for containers do not expand correctly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/43f9a94a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/43f9a94a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/43f9a94a
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 43f9a94ad364f691069723667983f1b0ed285a55
Parents: 772759c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 25 23:55:23 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 25 23:55:23 2014 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/agent/ProcessHelper.py | 5 +++--
slider-agent/src/main/python/agent/main.py | 13 +++++++++----
.../org/apache/slider/providers/agent/AgentKeys.java | 2 +-
.../slider/providers/agent/AgentProviderService.java | 2 +-
4 files changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-agent/src/main/python/agent/ProcessHelper.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ProcessHelper.py b/slider-agent/src/main/python/agent/ProcessHelper.py
index 467c4d8..81737f4 100644
--- a/slider-agent/src/main/python/agent/ProcessHelper.py
+++ b/slider-agent/src/main/python/agent/ProcessHelper.py
@@ -24,11 +24,12 @@ import traceback
import sys
import posixpath
from shell import getTempFiles
+import Constants
logger = logging.getLogger()
-if 'AGENT_WORK_ROOT' in os.environ:
- pidfile = os.path.realpath(posixpath.join(os.environ['AGENT_WORK_ROOT'], "infra", "run", "agent.pid"))
+if Constants.AGENT_WORK_ROOT in os.environ:
+ pidfile = os.path.realpath(posixpath.join(os.environ[Constants.AGENT_WORK_ROOT], "infra", "run", "agent.pid"))
else:
pidfile = None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index a8fc4e1..48d84cd 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -43,7 +43,7 @@ formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
agentPid = os.getpid()
configFileRelPath = "infra/conf/agent.ini"
-logFileName = "agent.log"
+logFileName = "slider-agent.log"
SERVER_STATUS_URL="https://{0}:{1}{2}"
@@ -185,9 +185,14 @@ def main():
parser.add_option("--debug", dest="debug", help="Agent debug hint", default="")
(options, args) = parser.parse_args()
- if not 'AGENT_WORK_ROOT' in os.environ:
- parser.error("AGENT_WORK_ROOT environment variable must be set.")
- options.root_folder = os.environ['AGENT_WORK_ROOT']
+ if not Constants.AGENT_WORK_ROOT in os.environ and not 'PWD' in os.environ:
+ parser.error("AGENT_WORK_ROOT environment variable or PWD must be set.")
+ if Constants.AGENT_WORK_ROOT in os.environ:
+ options.root_folder = os.environ[Constants.AGENT_WORK_ROOT]
+ else:
+ # some launch environments do not end up setting all environment variables
+ options.root_folder = os.environ['PWD']
+
if not 'AGENT_LOG_ROOT' in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ['AGENT_LOG_ROOT']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 1da99b0..c271e8e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -91,7 +91,7 @@ public interface AgentKeys {
String CREATE_DEF_ZK_NODE = "create.default.zookeeper.node";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
- String AGENT_OUT_FILE = "agent.out";
+ String AGENT_OUT_FILE = "slider-agent.out";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/43f9a94a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f37150e..647c6a8 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -253,7 +253,7 @@ public class AgentProviderService extends AbstractProviderService implements
String workDir = ApplicationConstants.Environment.PWD.$();
launcher.setEnv("AGENT_WORK_ROOT", workDir);
log.info("AGENT_WORK_ROOT set to {}", workDir);
- String logDir = ApplicationConstants.Environment.LOG_DIRS.$();
+ String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
launcher.setEnv("AGENT_LOG_ROOT", logDir);
log.info("AGENT_LOG_ROOT set to {}", logDir);
if (System.getenv(HADOOP_USER_NAME) != null) {
[03/22] git commit: SLIDER-435. Slider client should accept target
yarn queue as an input parameter
Posted by sm...@apache.org.
SLIDER-435. Slider client should accept target yarn queue as an input parameter
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e3c49d59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e3c49d59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e3c49d59
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: e3c49d59f6b847b7fc36b7acc9f531782d8336e9
Parents: 0d41051
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Sep 21 17:23:18 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Sep 21 17:23:18 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/resources.json | 2 +-
.../org/apache/slider/api/InternalKeys.java | 4 +
.../org/apache/slider/client/SliderClient.java | 7 +-
.../common/SliderXMLConfKeysForTesting.java | 2 +
.../AbstractClusterBuildingActionArgs.java | 4 +
.../apache/slider/common/params/Arguments.java | 1 +
.../slider/core/build/InstanceBuilder.java | 16 +++
.../resources_queue_labels.json | 17 +++
.../providers/agent/TestBuildBasicAgent.groovy | 39 ++++++-
.../framework/AgentCommandTestBase.groovy | 36 ++++--
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 1 -
.../AppsThroughAgentQueueAndLabelsIT.groovy | 113 +++++++++++++++++++
.../clusters/remote/slider/slider-client.xml | 14 ++-
13 files changed, 244 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
index d2fdbd8..c3fec68 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -24,7 +24,7 @@
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
- "yarn.component.instances": "1",
+ "yarn.component.instances": "0",
"yarn.memory": "256"
},
"HBASE_THRIFT2": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index ffa3600..5f150e6 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -94,6 +94,10 @@ public interface InternalKeys {
*/
String KEYTAB_LOCATION = "internal.keytab.location";
+ /**
+ * Queue used to deploy the app: {@value}
+ */
+ String INTERNAL_QUEUE = "internal.queue";
/**
* Flag to indicate whether or not the chaos monkey is enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3df5b18..221fc43 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -783,6 +783,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
builder.propagateFilename();
builder.propagatePrincipals();
builder.setImageDetailsIfAvailable(buildInfo.getImage(), buildInfo.getAppHomeDir());
+ builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
if (SliderUtils.isUnset(quorum)) {
@@ -1262,7 +1263,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
-
+ String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
+ if(!SliderUtils.isUnset(suppliedQueue)) {
+ amQueue = suppliedQueue;
+ log.info("Using queue {} for the application instance.", amQueue);
+ }
amLauncher.setQueue(amQueue);
// Submit the application to the applications manager
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index 2cb6594..4c56240 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -60,6 +60,8 @@ public interface SliderXMLConfKeysForTesting {
int DEFAULT_ACCUMULO_LIVE_TIME_SECONDS = 90;
String KEY_TEST_AGENT_ENABLED = "slider.test.agent.enabled";
+ String KEY_AGENTTESTS_QUEUE_LABELED_DEFINED = "slider.test.agent.labeled.queue.enabled";
+ String KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED = "slider.test.agent.labels.defined";
int DEFAULT_AGENT_LAUNCH_TIME_SECONDS = 60 * 3;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index fa71677..56e01c3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -93,6 +93,10 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Template application configuration")
public File template;
+ @Parameter(names = {ARG_QUEUE},
+ description = "Queue to submit the application")
+ public String queue;
+
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 7f8aa83..b119245 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -78,6 +78,7 @@ public interface Arguments {
String ARG_ZKPORT = "--zkport";
String ARG_ZKHOSTS = "--zkhosts";
String ARG_REPLACE_PKG = "--replacepkg";
+ String ARG_QUEUE = "--queue";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 528a0c4..669a5ad 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -49,6 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
+import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE;
import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR;
import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR;
import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
@@ -149,6 +150,21 @@ public class InstanceBuilder {
}
/**
+ * Set the queue used to start the application
+ * @param queue
+ * @throws BadConfigException
+ */
+ public void setQueue(String queue) throws BadConfigException {
+ if(queue != null) {
+ if(SliderUtils.isUnset(queue)) {
+ throw new BadConfigException("Queue value cannot be empty.");
+ }
+
+ instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue);
+ }
+ }
+
+ /**
* Set up the image/app home path
* @param appImage path in the DFS to the tar file
* @param appHomeDir other strategy: home dir
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json b/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json
new file mode 100644
index 0000000..d121608
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json
@@ -0,0 +1,17 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "COMMAND_LOGGER": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.label.expression":"blue"
+ },
+ "slider-appmaster": {
+ "yarn.label.expression":"red"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 1b5269a..2f6a15e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.providers.agent
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.InternalKeys
import org.apache.slider.api.ResourceKeys
import org.apache.slider.api.RoleKeys
import org.apache.slider.client.SliderClient
@@ -354,7 +355,43 @@ class TestBuildBasicAgent extends AgentTestBase {
fail("Build operation should not fail")
}
}
-
+
+ @Test
+ public void testSubmitToSpecificQueue() throws Throwable {
+ String clustername = createMiniCluster(
+ "",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, "file://" + appDef.absolutePath,
+ ARG_RESOURCES, TEST_FILES + "good/resources.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json",
+ ARG_QUEUE, "labeled"
+ ],
+ true, false,
+ false)
+ } catch (BadConfigException exception) {
+ log.error(
+ "Build operation should not have failed with exception : \n$exception")
+ fail("Build operation should not fail")
+ }
+
+ AggregateConf instanceDefinition = loadInstanceDefinition(clustername)
+ def label = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_QUEUE)
+ assert label == "labeled", "Expect labeled as the queue"
+ }
+
@Test
public void testBadAgentArgs() throws Throwable {
String clustername = createMiniCluster(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index f1edff4..cf32e94 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -21,8 +21,10 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderUtils
import org.apache.tools.zip.ZipEntry
import org.apache.tools.zip.ZipOutputStream
import org.junit.Before
@@ -35,6 +37,8 @@ class AgentCommandTestBase extends CommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
public static final boolean AGENTTESTS_ENABLED
+ public static final boolean AGENTTESTS_QUEUE_LABELED_DEFINED
+ public static final boolean AGENTTESTS_LABELS_RED_BLUE_DEFINED
private static String TEST_APP_PKG_DIR_PROP = "test.app.pkg.dir"
private static String TEST_APP_PKG_FILE_PROP = "test.app.pkg.file"
private static String TEST_APP_PKG_NAME_PROP = "test.app.pkg.name"
@@ -57,6 +61,10 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
static {
AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
+ AGENTTESTS_QUEUE_LABELED_DEFINED =
+ SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_QUEUE_LABELED_DEFINED, false)
+ AGENTTESTS_LABELS_RED_BLUE_DEFINED =
+ SLIDER_CONFIG.getBoolean(KEY_AGENTTESTS_LABELS_RED_BLUE_DEFINED, false)
}
protected String getAppResource() {
@@ -74,6 +82,14 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assume(AGENTTESTS_ENABLED, "Agent tests disabled")
}
+ public static void assumeQueueNamedLabelDefined() {
+ assume(AGENTTESTS_QUEUE_LABELED_DEFINED, "Custom queue named labeled is not defined")
+ }
+
+ public static void assumeLabelsRedAndBlueAdded() {
+ assume(AGENTTESTS_LABELS_RED_BLUE_DEFINED, "Custom node labels not defined")
+ }
+
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
@@ -110,17 +126,23 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
public static void assertComponentCount(String component, int count, SliderShell shell) {
log.info("Asserting component count.")
+ int instanceCount = getComponentCount(component, shell)
+ assert count == instanceCount, 'Instance count for component did not match expected.'
+ }
+
+ public static int getComponentCount(String component, SliderShell shell) {
String entry = findLineEntry(shell, ["instances", component] as String[])
- log.info(entry)
- assert entry != null
int instanceCount = 0
- int index = entry.indexOf("container_")
- while (index != -1) {
- instanceCount++;
- index = entry.indexOf("container_", index + 1)
+ if (!SliderUtils.isUnset(entry)) {
+ log.info(entry)
+ int index = entry.indexOf("container_")
+ while (index != -1) {
+ instanceCount++;
+ index = entry.indexOf("container_", index + 1)
+ }
}
- assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
+ return instanceCount
}
public static String findLineEntry(SliderShell shell, String[] locaters) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 8930b8d..7e39791 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -50,7 +50,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, APPLICATION_NAME,
- //ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
new file mode 100644
index 0000000..f3f6612
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AppsThroughAgentQueueAndLabelsIT extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "happy-path-with-queue-labels"
+ private static String TARGET_QUEUE = "labeled"
+ private static String APP_RESOURCE4 =
+ "../slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json"
+
+ @After
+ public void destroyCluster() {
+ cleanup(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testCreateWithQueueAndLabels() throws Throwable {
+ assumeAgentTestsEnabled()
+ assumeQueueNamedLabelDefined()
+ assumeLabelsRedAndBlueAdded()
+
+ cleanup(APPLICATION_NAME)
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_QUEUE, TARGET_QUEUE,
+ ARG_TEMPLATE, APP_TEMPLATE,
+ ARG_RESOURCES, APP_RESOURCE4
+ ])
+
+ logShell(shell)
+
+ ensureApplicationIsUp(APPLICATION_NAME)
+
+ repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '1']);
+
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+ assertComponentCount(COMMAND_LOGGER, 1, shell)
+
+ //flex
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_FLEX,
+ APPLICATION_NAME,
+ ARG_COMPONENT,
+ COMMAND_LOGGER,
+ "2"])
+
+ // sleep till the new instance starts
+ repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
+
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+ assertComponentCount(COMMAND_LOGGER, 2, shell)
+
+ assertSuccess(shell)
+ assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+ }
+
+ boolean hasContainerCountExceeded(Map<String, String> args) {
+ int expectedCount = args['arg1'].toInteger();
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ //logShell(shell)
+ int instanceCount = getComponentCount(COMMAND_LOGGER, shell)
+ log.info("Noticed instance count - " + instanceCount + " for " + COMMAND_LOGGER)
+ if (instanceCount >= expectedCount) {
+ return true
+ }
+
+ return false
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c49d59/src/test/clusters/remote/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/remote/slider/slider-client.xml b/src/test/clusters/remote/slider/slider-client.xml
index 5ed4d10..ce261e9 100644
--- a/src/test/clusters/remote/slider/slider-client.xml
+++ b/src/test/clusters/remote/slider/slider-client.xml
@@ -53,7 +53,19 @@
<name>slider.test.agent.enabled</name>
<value>true</value>
</property>
-
+
+ <property>
+ <name>slider.test.agent.labeled.queue.enabled</name>
+ <value>false</value>
+ <description>Add a queue named labeled</description>
+ </property>
+
+ <property>
+ <name>slider.test.agent.labels.defined</name>
+ <value>false</value>
+ <description>Add node labels red and blue and ensure enough capacities are allocated</description>
+ </property>
+
<property>
<name>slider.test.agent.tar</name>
<value>hdfs://c6403.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value>
[16/22] git commit: SLIDER-450. The template and resources json files
embedded within the app package should be named default and include comments
to that effect
Posted by sm...@apache.org.
SLIDER-450. The template and resources json files embedded within the app package should be named default and include comments to that effect
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/772759c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/772759c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/772759c3
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 772759c305bd11e940a4fa0f4f2d7994d1581471
Parents: ed960d3
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 25 22:25:49 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 25 22:25:49 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/README.txt | 9 ++--
app-packages/app-pkg-template/README.txt | 7 ++-
.../app-pkg-template/appConfig-default.json | 21 +++++++++
app-packages/app-pkg-template/appConfig.json | 21 ---------
.../app-pkg-template/resources-default.json | 16 +++++++
app-packages/app-pkg-template/resources.json | 16 -------
app-packages/hbase-win/README.txt | 8 ++--
app-packages/hbase/README.txt | 7 ++-
app-packages/memcached-win/README.txt | 7 ++-
.../memcached-win/appConfig-default.json | 24 ++++++++++
app-packages/memcached-win/appConfig.json | 24 ----------
.../memcached-win/resources-default.json | 16 +++++++
app-packages/memcached-win/resources.json | 16 -------
app-packages/memcached/README.txt | 7 ++-
app-packages/memcached/appConfig-default.json | 24 ++++++++++
app-packages/memcached/appConfig.json | 24 ----------
app-packages/memcached/resources-default.json | 16 +++++++
app-packages/memcached/resources.json | 16 -------
app-packages/storm/README.txt | 7 ++-
app-packages/storm/appConfig-default.json | 46 ++++++++++++++++++++
app-packages/storm/appConfig.json | 46 --------------------
app-packages/storm/resources-default.json | 34 +++++++++++++++
app-packages/storm/resources.json | 34 ---------------
app-packages/storm/src/assembly/storm.xml | 4 +-
24 files changed, 222 insertions(+), 228 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/accumulo/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/README.txt b/app-packages/accumulo/README.txt
index f75814c..3a857ee 100644
--- a/app-packages/accumulo/README.txt
+++ b/app-packages/accumulo/README.txt
@@ -32,10 +32,9 @@ slider version, and in the second case it will match the pkg.version
Verify the content using
zip -Tv slider-accumulo-app-package*.zip
-While appConfig.json and resources.json are not required for the package they
-work well as the default configuration for Slider apps. So it is advisable that
-when you create an application package for Slider, include sample/default
-resources.json and appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
In the maven packaging case, the version of Accumulo used for the app package
can be adjusted by adding a flag such as
@@ -45,7 +44,7 @@ can be adjusted by adding a flag such as
package are designed for Accumulo 1.6.0 only and may need to be modified to be
applicable for other versions of the app package.
-Note also that the sample appConfig.json provided only works with Accumulo 1.6,
+Note also that the sample appConfig-default.json provided only works with Accumulo 1.6,
while for Accumulo 1.5 the instance.volumes property must be replaced with
instance.dfs.dir (and it cannot use the provided variable ${DEFAULT_DATA_DIR}
which is an HDFS URI).
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/app-pkg-template/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/README.txt b/app-packages/app-pkg-template/README.txt
index 00dfdbc..266f34f 100644
--- a/app-packages/app-pkg-template/README.txt
+++ b/app-packages/app-pkg-template/README.txt
@@ -28,7 +28,6 @@ Create a zip package at the root of the package (<slider enlistment>/app-package
Verify the content using
zip -Tv myapp-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a one-node Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/app-pkg-template/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/appConfig-default.json b/app-packages/app-pkg-template/appConfig-default.json
new file mode 100644
index 0000000..3908030
--- /dev/null
+++ b/app-packages/app-pkg-template/appConfig-default.json
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": "myapp-1.0.0.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/myapp-1.0.0",
+
+ "site.global.listen_port": "${MYAPP_COMPONENT.ALLOCATED_PORT}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ },
+ "MYAPP_COMPONENT": {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/app-pkg-template/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/appConfig.json b/app-packages/app-pkg-template/appConfig.json
deleted file mode 100644
index 3908030..0000000
--- a/app-packages/app-pkg-template/appConfig.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": "myapp-1.0.0.zip",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
-
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/myapp-1.0.0",
-
- "site.global.listen_port": "${MYAPP_COMPONENT.ALLOCATED_PORT}"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "MYAPP_COMPONENT": {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/app-pkg-template/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/resources-default.json b/app-packages/app-pkg-template/resources-default.json
new file mode 100644
index 0000000..0e95879
--- /dev/null
+++ b/app-packages/app-pkg-template/resources-default.json
@@ -0,0 +1,16 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "MYAPP_COMPONENT": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/app-pkg-template/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/resources.json b/app-packages/app-pkg-template/resources.json
deleted file mode 100644
index 0e95879..0000000
--- a/app-packages/app-pkg-template/resources.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- },
- "components": {
- "slider-appmaster": {
- },
- "MYAPP_COMPONENT": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/hbase-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
index 6865168..0e64a84 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -17,10 +17,10 @@
Create Slider App Package for HBase on Windows
-While appConfig.json and resources.json are not required for the package they
-work well as the default configuration for Slider apps. So it is advisable that
-when you create an application package for Slider, include sample/default
-resources.json and appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
+
OPTION-I: Use a downloaded hbase tarball fro Windows
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/hbase/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt
index d0e3067..1d59730 100644
--- a/app-packages/hbase/README.txt
+++ b/app-packages/hbase/README.txt
@@ -17,10 +17,9 @@
Create Slider App Package for HBase
-While appConfig.json and resources.json are not required for the package they
-work well as the default configuration for Slider apps. So it is advisable that
-when you create an application package for Slider, include sample/default
-resources.json and appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
OPTION-I: Use a downloaded tarball
OPTION-II: Use the tarball from the local repository
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached-win/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/README.txt b/app-packages/memcached-win/README.txt
index 4d93b91..84d2728 100644
--- a/app-packages/memcached-win/README.txt
+++ b/app-packages/memcached-win/README.txt
@@ -30,7 +30,6 @@ Create a zip package at the root of the package (<slider enlistment>/app-package
Verify the content using
unzip -l "$@" jmemcached-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached-win/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig-default.json b/app-packages/memcached-win/appConfig-default.json
new file mode 100644
index 0000000..059c712
--- /dev/null
+++ b/app-packages/memcached-win/appConfig-default.json
@@ -0,0 +1,24 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
+ "java_home": "C:\\java",
+
+ "site.global.app_user": "hadoop",
+ "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
+ "site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
+ "site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
+ "site.global.xmx_val": "256m",
+ "site.global.xms_val": "128m",
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
+
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
deleted file mode 100644
index 059c712..0000000
--- a/app-packages/memcached-win/appConfig.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
- "java_home": "C:\\java",
-
- "site.global.app_user": "hadoop",
- "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
- "site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
- "site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
- "site.global.xmx_val": "256m",
- "site.global.xms_val": "128m",
- "site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
-
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached-win/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/resources-default.json b/app-packages/memcached-win/resources-default.json
new file mode 100644
index 0000000..f0e02ac
--- /dev/null
+++ b/app-packages/memcached-win/resources-default.json
@@ -0,0 +1,16 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "MEMCACHED": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached-win/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/resources.json b/app-packages/memcached-win/resources.json
deleted file mode 100644
index f0e02ac..0000000
--- a/app-packages/memcached-win/resources.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- },
- "components": {
- "slider-appmaster": {
- },
- "MEMCACHED": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/memcached/README.txt b/app-packages/memcached/README.txt
index eed2954..d9d8810 100644
--- a/app-packages/memcached/README.txt
+++ b/app-packages/memcached/README.txt
@@ -29,7 +29,6 @@ Create a zip package at the root of the package (<slider enlistment>/app-package
Verify the content using
unzip -l "$@" jmemcached-1.0.0.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig-default.json b/app-packages/memcached/appConfig-default.json
new file mode 100644
index 0000000..7b4aa97
--- /dev/null
+++ b/app-packages/memcached/appConfig-default.json
@@ -0,0 +1,24 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
+ "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
+
+ "site.global.additional_cp": "/usr/lib/hadoop/lib/*",
+ "site.global.xmx_val": "256m",
+ "site.global.xms_val": "128m",
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
deleted file mode 100644
index 7b4aa97..0000000
--- a/app-packages/memcached/appConfig.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
-
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
- "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
-
- "site.global.additional_cp": "/usr/lib/hadoop/lib/*",
- "site.global.xmx_val": "256m",
- "site.global.xms_val": "128m",
- "site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/resources-default.json b/app-packages/memcached/resources-default.json
new file mode 100644
index 0000000..f0e02ac
--- /dev/null
+++ b/app-packages/memcached/resources-default.json
@@ -0,0 +1,16 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "MEMCACHED": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/memcached/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/resources.json b/app-packages/memcached/resources.json
deleted file mode 100644
index f0e02ac..0000000
--- a/app-packages/memcached/resources.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- },
- "components": {
- "slider-appmaster": {
- },
- "MEMCACHED": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/storm/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index df48305..12dc140 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -31,7 +31,6 @@ mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
App package can be found in
app-packages/storm/target/slider-storm-app-package-${pkg.version}.zip
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
+appConfig-default.json and resources-default.json are not required to be packaged.
+These files are included as reference configuration for Slider apps and are suitable
+for a one-node cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/storm/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig-default.json b/app-packages/storm/appConfig-default.json
new file mode 100644
index 0000000..683b8cc
--- /dev/null
+++ b/app-packages/storm/appConfig-default.json
@@ -0,0 +1,46 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/STORM/slider-storm-app-package-${pkg.version}.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "create.default.zookeeper.node": "true",
+ "system_configs": "core-site",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_id": "Application2",
+ "site.global.ganglia_enabled":"true",
+ "site.global.ganglia_server_port": "8668",
+ "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
+ "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
+
+ "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
+ "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
+ "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
+ "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
+ "site.storm-site.transactional.zookeeper.root": "/transactional",
+ "site.storm-site.storm.zookeeper.port": "2181",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
+ "site.storm-site.drpc.invocations.port": "0",
+ "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
+ "site.storm-site.transactional.zookeeper.port": "null",
+ "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
+ "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.drpc.port": "0",
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
deleted file mode 100644
index 683b8cc..0000000
--- a/app-packages/storm/appConfig.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/STORM/slider-storm-app-package-${pkg.version}.zip",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "create.default.zookeeper.node": "true",
- "system_configs": "core-site",
-
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_id": "Application2",
- "site.global.ganglia_enabled":"true",
- "site.global.ganglia_server_port": "8668",
- "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
- "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
-
- "site.storm-site.storm.log.dir" : "${AGENT_LOG_ROOT}",
- "site.storm-site.storm.zookeeper.servers": "['${ZK_HOST}']",
- "site.storm-site.nimbus.thrift.port": "${NIMBUS.ALLOCATED_PORT}",
- "site.storm-site.storm.local.dir": "${AGENT_WORK_ROOT}/app/tmp/storm",
- "site.storm-site.transactional.zookeeper.root": "/transactional",
- "site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
- "site.storm-site.dev.zookeeper.path": "${AGENT_WORK_ROOT}/app/tmp/dev-storm-zookeeper",
- "site.storm-site.drpc.invocations.port": "0",
- "site.storm-site.storm.zookeeper.root": "${DEF_ZK_PATH}",
- "site.storm-site.transactional.zookeeper.port": "null",
- "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
- "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
- "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
- "site.storm-site.drpc.port": "0",
- "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/storm/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources-default.json b/app-packages/storm/resources-default.json
new file mode 100644
index 0000000..acecd91
--- /dev/null
+++ b/app-packages/storm/resources-default.json
@@ -0,0 +1,34 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "NIMBUS": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1"
+ },
+ "STORM_UI_SERVER": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1"
+ },
+ "DRPC_SERVER": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1"
+ },
+ "SUPERVISOR": {
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "1"
+ },
+ "STORM_REST_API": {
+ "yarn.role.priority": "5",
+ "yarn.component.instances": "0"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
deleted file mode 100644
index acecd91..0000000
--- a/app-packages/storm/resources.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- "yarn.log.include.patterns": "",
- "yarn.log.exclude.patterns": "",
- "yarn.log.interval": "0"
- },
- "components": {
- "slider-appmaster": {
- },
- "NIMBUS": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1"
- },
- "STORM_UI_SERVER": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1"
- },
- "DRPC_SERVER": {
- "yarn.role.priority": "3",
- "yarn.component.instances": "1"
- },
- "SUPERVISOR": {
- "yarn.role.priority": "4",
- "yarn.component.instances": "1"
- },
- "STORM_REST_API": {
- "yarn.role.priority": "5",
- "yarn.component.instances": "0"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/772759c3/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
index 47d223a..cc000c5 100644
--- a/app-packages/storm/src/assembly/storm.xml
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -30,7 +30,7 @@
<files>
<file>
- <source>appConfig.json</source>
+ <source>appConfig-default.json</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
<fileMode>0755</fileMode>
@@ -57,7 +57,7 @@
<exclude>pom.xml</exclude>
<exclude>src/**</exclude>
<exclude>target/**</exclude>
- <exclude>appConfig.json</exclude>
+ <exclude>appConfig-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
[22/22] git commit: SLIDER-158. Add a slider diagnostics command
(Thomas Liu via smohanty)
Posted by sm...@apache.org.
SLIDER-158. Add a slider diagnostics command (Thomas Liu via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f8210a7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f8210a7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f8210a7c
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: f8210a7c00c54054dc690abbe5088cd4ddea4e3e
Parents: ce647d3
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Sep 27 00:22:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Sep 27 00:22:26 2014 -0700
----------------------------------------------------------------------
slider-core/pom.xml | 6 +
.../org/apache/slider/client/SliderClient.java | 280 ++++++++++++++++++-
.../common/params/ActionDiagnosticArgs.java | 66 +++++
.../apache/slider/common/params/Arguments.java | 7 +
.../apache/slider/common/params/ClientArgs.java | 11 +-
.../slider/common/params/SliderActions.java | 4 +
.../apache/slider/common/tools/SliderUtils.java | 108 +++++++
.../slider/providers/agent/AgentKeys.java | 5 +
8 files changed, 485 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 39c17c6..7af8bd7 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -448,6 +448,12 @@
</dependency>
<dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>1.1</version>
+ </dependency>
+
+ <dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-core</artifactId>
<version>1.5</version>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d57f7dc..06c37ba 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,6 +19,7 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -33,6 +34,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -48,6 +51,7 @@ import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
@@ -91,6 +95,7 @@ import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.persist.ConfPersister;
+import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.registry.YARNRegistryClient;
import org.apache.slider.core.registry.docstore.ConfigFormat;
@@ -115,9 +120,15 @@ import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
@@ -125,6 +136,7 @@ import java.io.IOException;
import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -338,6 +350,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (ACTION_VERSION.equals(action)) {
exitCode = actionVersion();
+ } else if (ACTION_DIAGNOSTIC.equals(action)) {
+ exitCode = actionDiagnostic(serviceArgs.getActionDiagnosticArgs());
} else {
throw new SliderException(EXIT_UNIMPLEMENTED,
"Unimplemented: " + action);
@@ -346,7 +360,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return exitCode;
}
- /**
+/**
* Perform everything needed to init the hadoop binding.
* This assumes that the service is already in inited or started state
* @throws IOException
@@ -2386,6 +2400,268 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return sids;
}
+ /**
+ * diagnostic operation
+ *
+ * @param clusterName
+ * application name
+ * @param diagosticArgs
+ * diagnostic Arguments
+ * @return 0 for success, -1 for some issues that aren't errors, just
+ * failures to retrieve information (e.g. no application name
+ * specified)
+ * @throws YarnException
+ * YARN problems
+ * @throws IOException
+ * Network or other problems
+ */
+ private int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
+ try {
+ if (diagnosticArgs.client) {
+ actionDiagnosticClient();
+ } else if (SliderUtils.isSet(diagnosticArgs.application)) {
+ actionDiagnosticApplication(diagnosticArgs);
+ } else if (SliderUtils.isSet(diagnosticArgs.slider)) {
+ actionDiagnosticSlider(diagnosticArgs);
+ } else if (diagnosticArgs.yarn) {
+ actionDiagnosticYarn(diagnosticArgs);
+ } else if (diagnosticArgs.credentials) {
+ actionDiagnosticCredentials();
+ } else if (SliderUtils.isSet(diagnosticArgs.all)) {
+ actionDiagnosticAll(diagnosticArgs);
+ } else if (SliderUtils.isSet(diagnosticArgs.level)) {
+ actionDiagnosticIntelligent(diagnosticArgs);
+ } else {
+ // it's an unknown command
+ log.info(ActionDiagnosticArgs.USAGE);
+ return EXIT_USAGE;
+ }
+ } catch (Exception e) {
+ log.error(e.toString());
+ return EXIT_FALSE;
+ }
+ return EXIT_SUCCESS;
+ }
+
+ private void actionDiagnosticIntelligent(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException, URISyntaxException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.level;
+
+ try {
+ SliderUtils.validateClientConfigFile();
+ log.info("Slider-client.xml is accessible");
+ } catch (IOException e) {
+ // we are catching exceptions here because those are indication of
+ // validation result, and we need to print them here
+ log.error("validation of slider-client.xml fails because: "
+ + e.toString());
+ return;
+ }
+ SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
+ // cluster not found exceptions will be thrown upstream
+ ClusterDescription clusterDescription = clusterOperations
+ .getClusterDescription();
+ log.info("Slider AppMaster is accessible");
+
+ if (clusterDescription.state == ClusterDescription.STATE_LIVE) {
+ AggregateConf instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ String imagePath = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ //if null, that means slider uploaded the agent tarball for the user
+ //and we need to use where slider has put
+ if(imagePath == null){
+ ApplicationReport appReport = YARNRegistryClient.findInstance(clusterName);
+ Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/am");
+ imagePath = subPath.toString();
+ }
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem, imagePath);
+ log.info("Slider agent tarball is properly installed");
+ } catch (IOException e) {
+ log.error("can not find or open agent tar ball: " + e.toString());
+ return;
+ }
+ String pkgTarballPath = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ try {
+ SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
+ log.info("Application tarball is properly installed");
+ } catch (IOException e) {
+ log.error("can not find or open application tar ball: "
+ + e.toString());
+ return;
+ }
+ }
+ }
+
+ private void actionDiagnosticAll(ActionDiagnosticArgs diagnosticArgs)
+ throws IOException, YarnException {
+ //assign application name from param to each sub diagnostic function
+ diagnosticArgs.application = diagnosticArgs.all;
+ diagnosticArgs.slider = diagnosticArgs.all;
+ actionDiagnosticClient();
+ actionDiagnosticApplication(diagnosticArgs);
+ actionDiagnosticSlider(diagnosticArgs);
+ actionDiagnosticYarn(diagnosticArgs);
+ actionDiagnosticCredentials();
+ }
+
+ private void actionDiagnosticCredentials() throws BadConfigException, IOException
+ {
+ if (SliderUtils.isHadoopClusterSecure(SliderUtils
+ .loadClientConfigurationResource())) {
+ String credentialCacheFileDescription = null;
+ try {
+ credentialCacheFileDescription = SliderUtils
+ .checkCredentialCacheFile();
+ } catch (BadConfigException e) {
+ log.error("The credential config is not valid: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Unable to read the credential file: " + e.toString());
+ throw e;
+ }
+ log.info("Credential cache file for the current user: "
+ + credentialCacheFileDescription);
+ } else {
+ log.info("the cluster is not in secure mode");
+ }
+ }
+
+ private void actionDiagnosticYarn(ActionDiagnosticArgs diagnosticArgs) throws IOException, YarnException {
+ JSONObject converter = null;
+ log.info("the node in the YARN cluster has below state: ");
+ List<NodeReport> yarnClusterInfo;
+ try {
+ yarnClusterInfo = yarnClient.getNodeReports(NodeState.RUNNING);
+ } catch (YarnException e1) {
+ log.error("Exception happened when fetching node report from the YARN cluster: " + e1.toString());
+ throw e1;
+ } catch (IOException e1) {
+ log.error("Network problem happened when fetching node report YARN cluster: " + e1.toString());
+ throw e1;
+ }
+ for(NodeReport nodeReport : yarnClusterInfo){
+ log.info(nodeReport.toString());
+ }
+
+ if (diagnosticArgs.verbose) {
+ Writer configWriter = new StringWriter();
+ try {
+ Configuration.dumpConfiguration(yarnClient.getConfig(), configWriter);
+ } catch (IOException e1) {
+ log.error("Network problem happened when retrieving YARN config from YARN: " + e1.toString());
+ throw e1;
+ }
+ try {
+ converter = new JSONObject(configWriter.toString());
+ log.info("the configuration of the YARN cluster is: "
+ + converter.toString(2));
+
+ } catch (JSONException e) {
+ log.error("JSONException happened during parsing response from YARN: " + e.toString());
+ }
+ }
+ }
+
+ private void actionDiagnosticSlider(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException
+ {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.slider;
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ }
+ String imagePath = instanceDefinition.getInternalOperations().get(
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ //if null, it will be uploaded by Slider and thus at slider's path
+ if(imagePath == null){
+ ApplicationReport appReport = YARNRegistryClient.findInstance(clusterName);
+ Path path1 = sliderFileSystem.getTempPathForCluster(clusterName);
+ Path subPath = new Path(path1, appReport.getApplicationId().toString() + "/am");
+ imagePath = subPath.toString();
+ }
+ log.info("The path of slider agent tarball on HDFS is: " + imagePath);
+ }
+
+ private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs) throws YarnException, IOException
+ {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.application;
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations
+ .getInstanceDefinition();
+ } catch (YarnException e) {
+ log.error("Exception happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error("Network problem happened when retrieving instance definition from YARN: " + e.toString());
+ throw e;
+ }
+ String clusterDir = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_ROOT);
+ String pkgTarball = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_DEF);
+ String runAsUser = instanceDefinition.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.RUNAS_USER);
+
+ log.info("The location of the cluster instance directory in HDFS is: "
+ + clusterDir);
+ log.info("The name of the application package tarball on HDFS is: "
+ + pkgTarball);
+ log.info("The runas user of the application in the cluster is: "
+ + runAsUser);
+
+ if (diagnosticArgs.verbose) {
+ log.info("App config of the application: "
+ + instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application: "
+ + instanceDefinition.getResources().toJson());
+ }
+ }
+
+ private void actionDiagnosticClient() throws SliderException, IOException {
+ String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ SliderVersionInfo.loadAndPrintVersionInfo(log);
+ String clientConfigPath = SliderUtils.getClientConfigPath();
+ String jdkInfo = SliderUtils.getJDKInfo();
+ log.info("The slider command path: " + currentCommandPath);
+ log.info("The slider-client.xml used by current running command path: "
+ + clientConfigPath);
+ log.info(jdkInfo);
+
+ try {
+ SliderUtils.validateSliderClientEnvironment(log);
+ } catch (SliderException e) {
+ log.error(e.toString());
+ throw e;
+ } catch (IOException e) {
+ log.error(e.toString());
+ throw e;
+ }
+ }
+
private void logInstance(ServiceInstanceData instance,
boolean verbose) {
if (!verbose) {
@@ -2612,3 +2888,5 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
System.out.append(src);
}
}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
new file mode 100644
index 0000000..b2fde07
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
@@ -0,0 +1,66 @@
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_DIAGNOSTIC},
+commandDescription = SliderActions.DESCRIBE_ACTION_DIAGNOSTIC)
+public class ActionDiagnosticArgs extends AbstractActionArgs
+{
+ public static final String USAGE =
+ "Usage: " + SliderActions.ACTION_DIAGNOSTIC
+ + Arguments.ARG_CLIENT + "| "
+ + Arguments.ARG_SLIDER + " <appname> " + "| "
+ + Arguments.ARG_APPLICATION + " <appname> " + "| "
+ + Arguments.ARG_YARN + "| "
+ + Arguments.ARG_CREDENTIALS + "| "
+ + Arguments.ARG_ALL + " <appname> " + "| "
+ + Arguments.ARG_LEVEL + " <appname> "
+ + " [" + Arguments.ARG_VERBOSE + "] ";
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_DIAGNOSTIC;
+ }
+
+ @Parameter(names = {ARG_CLIENT},
+ description = "print configuration of the slider client")
+ public boolean client = false;
+
+ @Parameter(names = {ARG_SLIDER},
+ description = "print configuration of the running slider app master")
+ public String slider;
+
+ @Parameter(names = {ARG_APPLICATION},
+ description = "print configuration of the running application")
+ public String application;
+
+ @Parameter(names = {ARG_VERBOSE},
+ description = "print out information in details")
+ public boolean verbose = false;
+
+ @Parameter(names = {ARG_YARN},
+ description = "print configuration of the YARN cluster")
+ public boolean yarn = false;
+
+ @Parameter(names = {ARG_CREDENTIALS},
+ description = "print credentials of the current user")
+ public boolean credentials = false;
+
+ @Parameter(names = {ARG_ALL},
+ description = "print all of the information above")
+ public String all;
+
+ @Parameter(names = {ARG_LEVEL},
+ description = "diagnoze the application intelligently")
+ public String level;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index b119245..2b45ce8 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -78,6 +78,13 @@ public interface Arguments {
String ARG_ZKPORT = "--zkport";
String ARG_ZKHOSTS = "--zkhosts";
String ARG_REPLACE_PKG = "--replacepkg";
+ String ARG_CLIENT = "--client";
+ String ARG_SLIDER = "--slider";
+ String ARG_APPLICATION = "--application";
+ String ARG_YARN = "--yarn";
+ String ARG_CREDENTIALS = "--credentials";
+ String ARG_ALL = "--all";
+ String ARG_LEVEL = "--level";
String ARG_QUEUE = "--queue";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 7173a85..cd981b1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -62,6 +62,7 @@ public class ClientArgs extends CommonArgs {
private final ActionThawArgs actionThawArgs = new ActionThawArgs();
private final ActionVersionArgs actionVersionArgs = new ActionVersionArgs();
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
+ private final ActionDiagnosticArgs actionDiagnosticArgs = new ActionDiagnosticArgs();
public ClientArgs(String[] args) {
@@ -92,7 +93,8 @@ public class ClientArgs extends CommonArgs {
actionThawArgs,
actionHelpArgs,
actionVersionArgs,
- actionInstallPackageArgs
+ actionInstallPackageArgs,
+ actionDiagnosticArgs
);
}
@@ -112,6 +114,10 @@ public class ClientArgs extends CommonArgs {
}
}
+ public ActionDiagnosticArgs getActionDiagnosticArgs() {
+ return actionDiagnosticArgs;
+ }
+
public AbstractClusterBuildingActionArgs getBuildingActionArgs() {
return buildingActionArgs;
}
@@ -240,6 +246,9 @@ public class ClientArgs extends CommonArgs {
} else if (SliderActions.ACTION_VERSION.equals(action)) {
bindCoreAction(actionVersionArgs);
+ } else if (SliderActions.ACTION_DIAGNOSTIC.equals(action)) {
+ bindCoreAction(actionDiagnosticArgs);
+
} else if (action == null || action.isEmpty()) {
throw new BadCommandArgumentsException(ErrorStrings.ERROR_NO_ACTION);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 964f184..29fd098 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -44,6 +44,7 @@ public interface SliderActions {
String ACTION_THAW = "start";
String ACTION_USAGE = "usage";
String ACTION_VERSION = "version";
+ String ACTION_DIAGNOSTIC = "diagnostic";
String ACTION_INSTALL_PACKAGE = "install-package";
String DESCRIBE_ACTION_AM_SUICIDE =
"Tell the Slider Application Master to simulate a process failure by terminating itself";
@@ -78,4 +79,7 @@ public interface SliderActions {
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install the application package in the home directory under sub-folder packages";
+ String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
+
}
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 263460d..5313ab9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -19,6 +19,7 @@
package org.apache.slider.common.tools;
import com.google.common.base.Preconditions;
+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.output.ByteArrayOutputStream;
@@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.Slider;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderKeys;
@@ -76,6 +78,8 @@ import java.io.StringWriter;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
@@ -1780,4 +1784,108 @@ public final class SliderUtils {
execCommand(OPENSSL, 0, 5000, logger, "OpenSSL", OPENSSL, "version");
execCommand(PYTHON, 0, 5000, logger, "Python", PYTHON, "--version");
}
+
+ /**
+ * return the path to the currently running slider command
+ *
+ * @throws NullPointerException
+ * - If the pathname argument is null
+ * @throws SecurityException
+ * - if a security manager exists and its checkPermission method
+ * doesn't allow getting the ProtectionDomain
+ */
+ public static String getCurrentCommandPath() {
+ File f = new File(Slider.class.getProtectionDomain().getCodeSource()
+ .getLocation().getPath());
+ return f.getAbsolutePath();
+ }
+
+ /**
+ * return the path to the slider-client.xml used by the current running
+ * slider command
+ *
+ * @throws SecurityException
+ * - if a security manager exists and its checkPermission method
+ * denies access to the class loader for the class
+ */
+ public static String getClientConfigPath() {
+ URL path = ConfigHelper.class.getClassLoader().getResource(
+ SliderKeys.CLIENT_RESOURCE);
+ return path.toString();
+ }
+
+ /**
+ * validate if slider-client.xml under the path can be opened
+ *
+ * @throws IOException
+ * : the file can't be found or open
+ */
+ public static void validateClientConfigFile() throws IOException {
+ URL resURL = SliderVersionInfo.class.getClassLoader().getResource(
+ SliderKeys.CLIENT_RESOURCE);
+ if (resURL == null) {
+ throw new IOException(
+ "slider-client.xml doesn't exist on the path: "
+ + getClientConfigPath());
+ }
+
+ try {
+ InputStream inStream = resURL.openStream();
+ if (inStream == null) {
+ throw new IOException("slider-client.xml can't be opened");
+ }
+ } catch (IOException e) {
+ throw new IOException("slider-client.xml can't be opened: "
+ + e.toString());
+ }
+ }
+
+ /**
+ * validate if a file on HDFS can be open
+ *
+ * @throws IOException
+ * : the file can't be found or open
+ * @throws URISyntaxException
+ */
+ public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException{
+ URI pathURI = new URI(pathStr);
+ InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(pathURI));
+ if(inputStream == null){
+ throw new IOException("HDFS file " + pathStr + " can't be opened");
+ }
+ }
+
+ /**
+ * return the version and path of the JDK invoking the current running
+ * slider command
+ *
+ * @throws SecurityException
+ * - if a security manager exists and its checkPropertyAccess
+ * method doesn't allow access to the specified system property.
+ */
+ public static String getJDKInfo() {
+ String version = System.getProperty("java.version");
+ String javaHome = System.getProperty("java.home");
+ return "The version of the JDK invoking the current running slider command: "
+ + version + "; The path to it is: " + javaHome;
+ }
+
+ /**
+ * return a description of whether the current user has created credential
+ * cache files from kerberos servers
+ *
+ * @throws IOException
+ * @throws BadConfigException
+ * @throws SecurityException
+ * - if a security manager exists and its checkPropertyAccess
+ * method doesn't allow access to the specified system property.
+ */
+ public static String checkCredentialCacheFile() throws IOException,
+ BadConfigException {
+ String result = null;
+ if (!Shell.WINDOWS) {
+ result = Shell.execCommand("klist");
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8210a7c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index c271e8e..b30c18c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -60,6 +60,11 @@ public interface AgentKeys {
* Execution home for the agent.
*/
String APP_HOME = "app.home";
+ String APP_ROOT = "site.global.app_root";
+ /**
+ * Runas user of the application
+ */
+ String RUNAS_USER = "site.global.app_user";
/**
* Name of the service.
*/
[06/22] git commit: SLIDER-180. Changes to allow auto-build of hbase
win package (added missing property)
Posted by sm...@apache.org.
SLIDER-180. Changes to allow auto-build of hbase win package (added missing property)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e7c78605
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e7c78605
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e7c78605
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: e7c78605e768e5a63326c2631e0f86e00368729d
Parents: a1086e0
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Sep 23 06:58:35 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Sep 23 06:58:35 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/appConfig.json | 1 +
.../org/apache/slider/providers/agent/AgentProviderService.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7c78605/app-packages/hbase-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json
index 6f07d34..5b97e69 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -5,6 +5,7 @@
"global": {
"application.def": ".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
"create.default.zookeeper.node": "true",
+ "java_home": "C:\\java",
"site.global.app_user": "hadoop",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e7c78605/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bb98588..6976f50 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -567,8 +567,8 @@ public class AgentProviderService extends AbstractProviderService implements
response.setRestartEnabled(true);
}
} catch (SliderException e) {
- componentStatus.applyCommandResult(CommandResult.FAILED, command);
log.warn("Component instance failed operation.", e);
+ componentStatus.applyCommandResult(CommandResult.FAILED, command);
}
log.debug("Heartbeat response: " + response);
[14/22] git commit: SLIDER-450 Rename template and resources json
files to -default
Posted by sm...@apache.org.
SLIDER-450 Rename template and resources json files to -default
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5470c47a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5470c47a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5470c47a
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 5470c47aae8af30e62404a69cce6fc3e573efa24
Parents: a34837c
Author: tedyu <yu...@gmail.com>
Authored: Wed Sep 24 15:38:57 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Sep 24 15:38:57 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/appConfig-default.json | 38 ++++++++++++++++
app-packages/hbase-win/appConfig.json | 38 ----------------
app-packages/hbase-win/resources-default.json | 24 ++++++++++
app-packages/hbase-win/resources.json | 24 ----------
app-packages/hbase-win/src/assembly/hbase.xml | 4 +-
app-packages/hbase/appConfig-default.json | 47 ++++++++++++++++++++
app-packages/hbase/appConfig.json | 47 --------------------
app-packages/hbase/pom.xml | 4 +-
app-packages/hbase/resources-default.json | 36 +++++++++++++++
app-packages/hbase/resources.json | 36 ---------------
app-packages/hbase/src/assembly/hbase.xml | 4 +-
.../src/test/resources/resources-default.json | 24 ++++++++++
.../hbase/src/test/resources/resources.json | 24 ----------
13 files changed, 175 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig-default.json b/app-packages/hbase-win/appConfig-default.json
new file mode 100644
index 0000000..5b97e69
--- /dev/null
+++ b/app-packages/hbase-win/appConfig-default.json
@@ -0,0 +1,38 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
+ "create.default.zookeeper.node": "true",
+ "java_home": "C:\\java",
+
+ "site.global.app_user": "hadoop",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.hbase_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.global.monitor_protocol": "http",
+ "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
+ "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
+
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "yarn",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json
deleted file mode 100644
index 5b97e69..0000000
--- a/app-packages/hbase-win/appConfig.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/HBASE/slider-hbase-app-win-package-${bld.version}.zip",
- "create.default.zookeeper.node": "true",
- "java_home": "C:\\java",
-
- "site.global.app_user": "hadoop",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
- "site.global.hbase_instance_name": "instancename",
- "site.global.hbase_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "http",
- "site.global.hbase_additional_cp": "c:\\java\\lib\\tools.jar;",
- "site.global.java_library_path": "c:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\bin",
-
- "site.hbase-env.hbase_master_heapsize": "1024m",
- "site.hbase-env.hbase_regionserver_heapsize": "1024m",
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0",
- "site.hbase-site.hbase.master.port": "0"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/resources-default.json b/app-packages/hbase-win/resources-default.json
new file mode 100644
index 0000000..eb38726
--- /dev/null
+++ b/app-packages/hbase-win/resources-default.json
@@ -0,0 +1,24 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
+ },
+ "components": {
+ "HBASE_MASTER": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "slider-appmaster": {
+ },
+ "HBASE_REGIONSERVER": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/resources.json b/app-packages/hbase-win/resources.json
deleted file mode 100644
index eb38726..0000000
--- a/app-packages/hbase-win/resources.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "yarn.log.include.patterns": "",
- "yarn.log.exclude.patterns": "",
- "yarn.log.interval": "0"
- },
- "components": {
- "HBASE_MASTER": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "slider-appmaster": {
- },
- "HBASE_REGIONSERVER": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase-win/src/assembly/hbase.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/src/assembly/hbase.xml b/app-packages/hbase-win/src/assembly/hbase.xml
index e62da84..a94c827 100644
--- a/app-packages/hbase-win/src/assembly/hbase.xml
+++ b/app-packages/hbase-win/src/assembly/hbase.xml
@@ -30,7 +30,7 @@
<files>
<file>
- <source>appConfig.json</source>
+ <source>appConfig-default.json</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
<fileMode>0755</fileMode>
@@ -57,7 +57,7 @@
<exclude>pom.xml</exclude>
<exclude>src/**</exclude>
<exclude>target/**</exclude>
- <exclude>appConfig.json</exclude>
+ <exclude>appConfig-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig-default.json b/app-packages/hbase/appConfig-default.json
new file mode 100644
index 0000000..78479db
--- /dev/null
+++ b/app-packages/hbase/appConfig-default.json
@@ -0,0 +1,47 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": ".slider/package/HBASE/slider-hbase-app-package-${bld.version}.zip",
+ "create.default.zookeeper.node": "true",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "system_configs": "core-site",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
+
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_port": "8667",
+ "site.global.ganglia_server_id": "Application1",
+ "site.global.ganglia_enabled":"true",
+
+ "site.global.hbase_instance_name": "instancename",
+ "site.global.hbase_root_password": "secret",
+ "site.global.user_group": "hadoop",
+ "site.global.security_enabled": "false",
+ "site.global.monitor_protocol": "http",
+ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+ "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+
+ "site.hbase-env.hbase_master_heapsize": "1024m",
+ "site.hbase-env.hbase_regionserver_heapsize": "1024m",
+
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
+ "site.hbase-site.hbase.superuser": "yarn",
+ "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
+ "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
+ "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
+ "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
+ "site.hbase-site.hbase.regionserver.info.port": "0",
+ "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
+ "site.hbase-site.hbase.regionserver.port": "0",
+ "site.hbase-site.hbase.master.port": "0"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
deleted file mode 100644
index 78479db..0000000
--- a/app-packages/hbase/appConfig.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/HBASE/slider-hbase-app-package-${bld.version}.zip",
- "create.default.zookeeper.node": "true",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "system_configs": "core-site",
-
- "site.global.app_user": "yarn",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${pkg.version}",
-
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.global.ganglia_enabled":"true",
-
- "site.global.hbase_instance_name": "instancename",
- "site.global.hbase_root_password": "secret",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.monitor_protocol": "http",
- "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
- "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
- "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
-
- "site.hbase-env.hbase_master_heapsize": "1024m",
- "site.hbase-env.hbase_regionserver_heapsize": "1024m",
-
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "${DEF_ZK_PATH}",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0",
- "site.hbase-site.hbase.master.port": "0"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 3d57ac5..519d7c1 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -162,8 +162,8 @@
</slider.bin.dir>
<test.app.pkg.dir>target</test.app.pkg.dir>
<test.app.pkg.file>${app.package.name}.zip</test.app.pkg.file>
- <test.app.resource>target/test-config/resources.json</test.app.resource>
- <test.app.template>target/${app.package.name}/appConfig.json</test.app.template>
+ <test.app.resource>target/test-config/resources-default.json</test.app.resource>
+ <test.app.template>target/${app.package.name}/appConfig-default.json</test.app.template>
</systemPropertyVariables>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources-default.json b/app-packages/hbase/resources-default.json
new file mode 100644
index 0000000..c3fec68
--- /dev/null
+++ b/app-packages/hbase/resources-default.json
@@ -0,0 +1,36 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "HBASE_MASTER": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "slider-appmaster": {
+ },
+ "HBASE_REGIONSERVER": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "HBASE_REST": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT": {
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "0",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT2": {
+ "yarn.role.priority": "5",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
deleted file mode 100644
index c3fec68..0000000
--- a/app-packages/hbase/resources.json
+++ /dev/null
@@ -1,36 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- },
- "components": {
- "HBASE_MASTER": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "slider-appmaster": {
- },
- "HBASE_REGIONSERVER": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "HBASE_REST": {
- "yarn.role.priority": "3",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "HBASE_THRIFT": {
- "yarn.role.priority": "4",
- "yarn.component.instances": "0",
- "yarn.memory": "256"
- },
- "HBASE_THRIFT2": {
- "yarn.role.priority": "5",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/src/assembly/hbase.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/src/assembly/hbase.xml b/app-packages/hbase/src/assembly/hbase.xml
index e62da84..a94c827 100644
--- a/app-packages/hbase/src/assembly/hbase.xml
+++ b/app-packages/hbase/src/assembly/hbase.xml
@@ -30,7 +30,7 @@
<files>
<file>
- <source>appConfig.json</source>
+ <source>appConfig-default.json</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
<fileMode>0755</fileMode>
@@ -57,7 +57,7 @@
<exclude>pom.xml</exclude>
<exclude>src/**</exclude>
<exclude>target/**</exclude>
- <exclude>appConfig.json</exclude>
+ <exclude>appConfig-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/src/test/resources/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/src/test/resources/resources-default.json b/app-packages/hbase/src/test/resources/resources-default.json
new file mode 100644
index 0000000..eb38726
--- /dev/null
+++ b/app-packages/hbase/src/test/resources/resources-default.json
@@ -0,0 +1,24 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
+ },
+ "components": {
+ "HBASE_MASTER": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "slider-appmaster": {
+ },
+ "HBASE_REGIONSERVER": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5470c47a/app-packages/hbase/src/test/resources/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/src/test/resources/resources.json b/app-packages/hbase/src/test/resources/resources.json
deleted file mode 100644
index eb38726..0000000
--- a/app-packages/hbase/src/test/resources/resources.json
+++ /dev/null
@@ -1,24 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "yarn.log.include.patterns": "",
- "yarn.log.exclude.patterns": "",
- "yarn.log.interval": "0"
- },
- "components": {
- "HBASE_MASTER": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- },
- "slider-appmaster": {
- },
- "HBASE_REGIONSERVER": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
- }
- }
-}
[09/22] git commit: SLIDER-453 AM queue processing goes live before
init complete
Posted by sm...@apache.org.
SLIDER-453 AM queue processing goes live before init complete
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/94df22c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/94df22c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/94df22c9
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 94df22c92e48859fbc4b3a560981a6a70cc64b54
Parents: 937e0a7
Author: Steve Loughran <st...@apache.org>
Authored: Tue Sep 23 21:59:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Sep 23 21:59:42 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 29 ++++++++++++++++++--
.../slider/server/appmaster/state/AppState.java | 5 +++-
2 files changed, 30 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94df22c9/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 08d0049..365eeab 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -283,6 +283,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private final AtomicBoolean amCompletionFlag = new AtomicBoolean(false);
+ /**
+ * Flag set during the init process
+ */
+ private final AtomicBoolean initCompleted = new AtomicBoolean(false);
+
private volatile boolean success = true;
/**
@@ -419,11 +424,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@Override
protected void serviceStart() throws Exception {
super.serviceStart();
+ }
+
+ /**
+ * Start the queue processing
+ */
+ private void startQueueProcessing() {
+ log.info("Queue Processing started");
executorService.execute(actionQueues);
executorService.execute(new QueueExecutor(this, actionQueues));
}
- /* =================================================================== */
+/* =================================================================== */
/* RunService methods called from ServiceLauncher */
/* =================================================================== */
@@ -801,7 +813,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// now do the registration
registerServiceInstance(clustername, appid);
+ // declare the cluster initialized
+ log.info("Application Master Initialization Completed");
+ initCompleted.set(true);
+
+ // start handling any scheduled events
+
+ startQueueProcessing();
// Start the Slider AM provider
+
sliderAMProvider.start();
// launch the real provider; this is expected to trigger a callback that
@@ -1210,8 +1230,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.error("Role instance {} failed ", ri);
}
- getProviderService().notifyContainerCompleted(containerId);
- queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ // known nodes trigger notifications
+ if(!result.unknownNode) {
+ getProviderService().notifyContainerCompleted(containerId);
+ queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
+ }
}
reviewRequestAndReleaseNodes("onContainersCompleted");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/94df22c9/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 57952e7..abb6fe8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1256,13 +1256,14 @@ public class AppState {
}
/**
- * This is a very small class to send a triple result back from
+ * This is a very small class to send a multiple result back from
* the completion operation
*/
public static class NodeCompletionResult {
public boolean surplusNode = false;
public RoleInstance roleInstance;
public boolean containerFailed;
+ public boolean unknownNode = false;
public String toString() {
@@ -1271,6 +1272,7 @@ public class AppState {
sb.append("surplusNode=").append(surplusNode);
sb.append(", roleInstance=").append(roleInstance);
sb.append(", containerFailed=").append(containerFailed);
+ sb.append(", unknownNode=").append(unknownNode);
sb.append('}');
return sb.toString();
}
@@ -1354,6 +1356,7 @@ public class AppState {
log.error("Notified of completed container {} that is not in the list" +
" of active or failed containers", containerId);
completionOfUnknownContainerEvent.incrementAndGet();
+ result.unknownNode = true;
}
}
[15/22] git commit: SLIDER-456 switch accumulo version to 1.6.1
Posted by sm...@apache.org.
SLIDER-456 switch accumulo version to 1.6.1
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ed960d33
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ed960d33
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ed960d33
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: ed960d33c4baa948e48a18c324556912e9d5517e
Parents: 5470c47
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu Sep 25 06:58:46 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Sep 25 06:58:46 2014 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ed960d33/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9dd400e..b31d476 100644
--- a/pom.xml
+++ b/pom.xml
@@ -139,7 +139,7 @@
<hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
<hbase.version>0.98.4-hadoop2</hbase.version>
- <accumulo.version>1.6.1-SNAPSHOT</accumulo.version>
+ <accumulo.version>1.6.1</accumulo.version>
<!--
artifact versions
[13/22] git commit: SLIDER-116. Log handling for long-lived
applications (Gour Saha via smohanty)
Posted by sm...@apache.org.
SLIDER-116. Log handling for long-lived applications (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a34837ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a34837ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a34837ce
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: a34837ce1d0431c960f956251eb57e23054253d1
Parents: 314c3cc
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 24 13:50:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 24 13:51:00 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/resources.json | 3 +
.../accumulo/src/test/resources/resources.json | 3 +
app-packages/hbase-win/resources.json | 3 +
.../hbase/src/test/resources/resources.json | 3 +
app-packages/storm/resources.json | 3 +
.../org/apache/slider/api/ResourceKeys.java | 3 +
.../org/apache/slider/client/SliderClient.java | 2 +
.../slider/core/launch/AbstractLauncher.java | 51 +++++++++
.../slider/core/launch/AppMasterLauncher.java | 4 +-
.../core/launch/TestAppMasterLauncher.java | 112 +++++++++++++++++++
10 files changed, 186 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/resources.json b/app-packages/accumulo/resources.json
index f876901..f9cae60 100644
--- a/app-packages/accumulo/resources.json
+++ b/app-packages/accumulo/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/accumulo/src/test/resources/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/resources.json b/app-packages/accumulo/src/test/resources/resources.json
index 2711ba3..cfd9b3f 100644
--- a/app-packages/accumulo/src/test/resources/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"ACCUMULO_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase-win/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/resources.json b/app-packages/hbase-win/resources.json
index e0ff26f..eb38726 100644
--- a/app-packages/hbase-win/resources.json
+++ b/app-packages/hbase-win/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/hbase/src/test/resources/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/src/test/resources/resources.json b/app-packages/hbase/src/test/resources/resources.json
index e0ff26f..eb38726 100644
--- a/app-packages/hbase/src/test/resources/resources.json
+++ b/app-packages/hbase/src/test/resources/resources.json
@@ -3,6 +3,9 @@
"metadata": {
},
"global": {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"HBASE_MASTER": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/app-packages/storm/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index 2805a3b..acecd91 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -3,6 +3,9 @@
"metadata" : {
},
"global" : {
+ "yarn.log.include.patterns": "",
+ "yarn.log.exclude.patterns": "",
+ "yarn.log.interval": "0"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index b542f1a..56961c9 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -118,4 +118,7 @@ public interface ResourceKeys {
*/
int DEFAULT_CONTAINER_FAILURE_THRESHOLD = 5;
+ String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
+ String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
+ String YARN_LOG_INTERVAL = "yarn.log.interval";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 129f851..afbb4a8 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1032,6 +1032,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
MapOperations sliderAMResourceComponent =
resourceOperations.getOrAddComponent(SliderKeys.COMPONENT_AM);
+ MapOperations resourceGlobalOptions = resourceOperations.getGlobalOptions();
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
@@ -1043,6 +1044,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
yarnClient,
clusterSecure,
sliderAMResourceComponent,
+ resourceGlobalOptions,
applicationTags);
ApplicationId appId = amLauncher.getApplicationId();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 366fc8b..d3bb34c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -19,6 +19,8 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
@@ -28,6 +30,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.slider.api.ResourceKeys;
@@ -70,6 +73,7 @@ public abstract class AbstractLauncher extends Configured {
new HashMap<String, ByteBuffer>();
// security
protected final Credentials credentials = new Credentials();
+ protected LogAggregationContext logAggregationContext;
protected AbstractLauncher(Configuration conf,
@@ -258,6 +262,53 @@ public abstract class AbstractLauncher extends Configured {
}
}
+ public void extractLogAggregationContext(Map<String, String> map) {
+ if (map != null) {
+ String logPatternSepStr = "\\|";
+ String logPatternJoinStr = "|";
+ MapOperations options = new MapOperations("", map);
+
+ List<String> logIncludePatterns = new ArrayList<String>();
+ String includePatternExpression = options.getOption(
+ ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
+ if (!includePatternExpression.isEmpty()) {
+ String[] includePatterns = includePatternExpression
+ .split(logPatternSepStr);
+ for (String includePattern : includePatterns) {
+ String trimmedIncludePattern = includePattern.trim();
+ if (!trimmedIncludePattern.isEmpty()) {
+ logIncludePatterns.add(trimmedIncludePattern);
+ }
+ }
+ }
+ String logIncludePattern = StringUtils.join(logIncludePatterns,
+ logPatternJoinStr);
+ log.info("Log include patterns: {}", logIncludePattern);
+
+ List<String> logExcludePatterns = new ArrayList<String>();
+ String excludePatternExpression = options.getOption(
+ ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
+ if (!excludePatternExpression.isEmpty()) {
+ String[] excludePatterns = excludePatternExpression
+ .split(logPatternSepStr);
+ for (String excludePattern : excludePatterns) {
+ String trimmedExcludePattern = excludePattern.trim();
+ if (!trimmedExcludePattern.isEmpty()) {
+ logExcludePatterns.add(trimmedExcludePattern);
+ }
+ }
+ }
+ String logExcludePattern = StringUtils.join(logExcludePatterns,
+ logPatternJoinStr);
+ log.info("Log exclude patterns: {}", logExcludePattern);
+
+ long logInterval = options
+ .getOptionInt(ResourceKeys.YARN_LOG_INTERVAL, 0);
+ log.info("Log interval: {}", logInterval);
+ logAggregationContext = LogAggregationContext.newInstance(
+ logIncludePattern, logExcludePattern, logInterval);
+ }
+ }
/**
* Utility method to set up the classpath
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index c9ac20c..f1eeabd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -77,6 +77,7 @@ public class AppMasterLauncher extends AbstractLauncher {
SliderYarnClientImpl yarnClient,
boolean secureCluster,
Map<String, String> options,
+ Map<String, String> resourceGlobalOptions,
Set<String> applicationTags
) throws IOException, YarnException {
super(conf, fs);
@@ -96,7 +97,7 @@ public class AppMasterLauncher extends AbstractLauncher {
submissionContext.setApplicationTags(applicationTags);
}
extractResourceRequirements(resource, options);
-
+ extractLogAggregationContext(resourceGlobalOptions);
}
public void setMaxAppAttempts(int maxAppAttempts) {
@@ -170,6 +171,7 @@ public class AppMasterLauncher extends AbstractLauncher {
//container requirements
submissionContext.setResource(resource);
+ submissionContext.setLogAggregationContext(logAggregationContext);
if (keepContainersOverRestarts) {
log.debug("Requesting cluster stays running over AM failure");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34837ce/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
new file mode 100644
index 0000000..ca1e9ec
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderKeys;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAppMasterLauncher {
+ SliderYarnClientImpl mockYarnClient;
+ YarnClientApplication yarnClientApp;
+ ApplicationSubmissionContext appSubmissionContext;
+ Set<String> tags = Collections.emptySet();
+
+ @Before
+ public void initialize() throws Exception {
+ mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
+ yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
+ appSubmissionContext = EasyMock
+ .createNiceMock(ApplicationSubmissionContext.class);
+ EasyMock.expect(yarnClientApp.getApplicationSubmissionContext())
+ .andReturn(appSubmissionContext).once();
+ EasyMock.expect(mockYarnClient.createApplication())
+ .andReturn(yarnClientApp).once();
+ }
+
+ @Test
+ public void testExtractLogAggregationContext() throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS,
+ " | slider*.txt |agent.out| |");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+ options.put(ResourceKeys.YARN_LOG_INTERVAL, "30");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
+ SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
+ tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "slider*.txt|agent.out";
+ Assert.assertEquals(expectedInclude,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+
+ String expectedExclude = "command*.json|agent.log*";
+ Assert.assertEquals(expectedExclude,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+
+ Assert.assertEquals(30,
+ appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+ @Test
+ public void testExtractLogAggregationContextEmptyIncludePattern()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+ options.put(ResourceKeys.YARN_LOG_INTERVAL, "600");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ AppMasterLauncher appMasterLauncher = new AppMasterLauncher("cl1",
+ SliderKeys.APP_TYPE, null, null, mockYarnClient, false, null, options,
+ tags);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "";
+ Assert.assertEquals(expectedInclude,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+
+ String expectedExclude = "command*.json|agent.log*";
+ Assert.assertEquals(expectedExclude,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+
+ Assert.assertEquals(600,
+ appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+}
[08/22] git commit: SLIDER-440. Add an env.sh file to limit post
install changes to conf dir (fix hdp-conf vs. conf)
Posted by sm...@apache.org.
SLIDER-440. Add an env.sh file to limit post install changes to conf dir (fix hdp-conf vs. conf)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/937e0a7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/937e0a7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/937e0a7c
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 937e0a7ce5941b88f024b3e7f6e6503f356274c2
Parents: 39e8365
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Sep 23 10:30:03 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Sep 23 10:30:03 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/conf-hdp/slider-client.xml | 28 ++++++++++++---------
slider-assembly/src/conf-hdp/slider-env.sh | 26 +++++++++++++++++++
2 files changed, 42 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/937e0a7c/slider-assembly/src/conf-hdp/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf-hdp/slider-client.xml b/slider-assembly/src/conf-hdp/slider-client.xml
index f844106..8e87e53 100644
--- a/slider-assembly/src/conf-hdp/slider-client.xml
+++ b/slider-assembly/src/conf-hdp/slider-client.xml
@@ -19,42 +19,46 @@
<!--
Properties set here are picked up in the client.
- They are not passed to the AM -though the filesystem
- binding details (URL And principal) are added to the
- hbase-site.xml file when a cluster is created.
-->
<configuration>
<property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
+ <name>slider.zookeeper.quorum</name>
+ <value>localhost:2181</value>
</property>
-
<property>
<name>slider.yarn.queue</name>
<value>default</value>
<description>YARN queue for the Application Master</description>
</property>
-
-
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+<!--
<property>
<name>yarn.application.classpath</name>
<value>
/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
</value>
</property>
-
-<!--
<property>
<name>yarn.resourcemanager.address</name>
- <value>master:8032</value>
+ <value>@RM-ADDR</value>
</property>
<property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>@RM-SCHEDULER-ADDR</value>
+ </property>
+
+ <property>
<name>fs.defaultFS</name>
- <value>hdfs://master:9090</value>
+ <value>@FILESYSTEM</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/937e0a7c/slider-assembly/src/conf-hdp/slider-env.sh
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf-hdp/slider-env.sh b/slider-assembly/src/conf-hdp/slider-env.sh
new file mode 100644
index 0000000..f3ec550
--- /dev/null
+++ b/slider-assembly/src/conf-hdp/slider-env.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# this is the shell script to start Slider deploying an application
+# Usage: slider <action> <commands>
+
+# The env variable SLIDER_JVM_OPTS can be used to override
+# the default JVM opts
+
+export JAVA_HOME=${JAVA_HOME}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
\ No newline at end of file
[07/22] git commit: SLIDER-452. App packages should have consistent
url keys
Posted by sm...@apache.org.
SLIDER-452. App packages should have consistent url keys
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/39e83651
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/39e83651
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/39e83651
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 39e83651afa65269bf4ab38f80ff1b356773c2a9
Parents: e7c7860
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Sep 23 07:33:35 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Sep 23 07:33:35 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/metainfo.xml | 4 ++--
app-packages/hbase-win/metainfo.xml | 4 ++--
app-packages/hbase/metainfo.xml | 10 +++++-----
app-packages/storm/metainfo.xml | 14 +++++++-------
4 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/39e83651/app-packages/accumulo/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml
index 30d0820..a926561 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -40,7 +40,7 @@
</value>
</export>
<export>
- <name>app.jmx</name>
+ <name>org.apache.slider.jmx</name>
<value>
${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml
</value>
@@ -93,7 +93,7 @@
<name>ACCUMULO_MONITOR</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
- <appExports>QuickLinks-app.jmx,QuickLinks-org.apache.slider.monitor</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports>
<commandScript>
<script>scripts/accumulo_monitor.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/39e83651/app-packages/hbase-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml
index c585720..17b4404 100644
--- a/app-packages/hbase-win/metainfo.xml
+++ b/app-packages/hbase-win/metainfo.xml
@@ -63,11 +63,11 @@
<appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports>
<componentExports>
<componentExport>
- <name>app.jmx</name>
+ <name>org.apache.slider.jmx</name>
<value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
</componentExport>
<componentExport>
- <name>app.monitor</name>
+ <name>org.apache.slider.monitor</name>
<value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
</componentExport>
</componentExports>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/39e83651/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index 1072a52..921ee48 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -54,11 +54,11 @@
<value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
</export>
<export>
- <name>app.metrics</name>
+ <name>org.apache.slider.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>app.ganglia</name>
+ <name>org.apache.slider.metrics.ui</name>
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
</exports>
@@ -82,14 +82,14 @@
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
<maxInstanceCount>2</maxInstanceCount>
- <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-app.metrics,QuickLinks-app.ganglia</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports>
<componentExports>
<componentExport>
- <name>app.jmx</name>
+ <name>org.apache.slider.jmx</name>
<value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
</componentExport>
<componentExport>
- <name>app.monitor</name>
+ <name>org.apache.slider.monitor</name>
<value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
</componentExport>
</componentExports>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/39e83651/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 2bf6ddc..bbb9db6 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -29,19 +29,19 @@
<name>QuickLinks</name>
<exports>
<export>
- <name>app.jmx</name>
+ <name>org.apache.slider.jmx</name>
<value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
</export>
<export>
- <name>app.monitor</name>
+ <name>org.apache.slider.monitor</name>
<value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
</export>
<export>
- <name>app.metrics</name>
+ <name>org.apache.slider.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>ganglia.ui</name>
+ <name>org.apache.slider.metrics.ui</name>
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
<export>
@@ -82,7 +82,7 @@
<name>NIMBUS</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-nimbus.host_port,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
+ <appExports>QuickLinks-nimbus.host_port,QuickLinks-org.apache.slider.metrics.ui,QuickLinks-org.apache.slider.metrics</appExports>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
@@ -111,7 +111,7 @@
<name>STORM_UI_SERVER</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
- <appExports>QuickLinks-app.monitor</appExports>
+ <appExports>QuickLinks-org.apache.slider.monitor</appExports>
<autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/ui_server.py</script>
@@ -135,7 +135,7 @@
<name>STORM_REST_API</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-app.jmx</appExports>
+ <appExports>QuickLinks-org.apache.slider.jmx</appExports>
<commandScript>
<script>scripts/rest_api.py</script>
<scriptType>PYTHON</scriptType>
[02/22] git commit: SLIDER-434. Modify default application package
locations to the default upload location
Posted by sm...@apache.org.
SLIDER-434. Modify default application package locations to the default upload location
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0d41051a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0d41051a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0d41051a
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 0d41051a40709071336e1f2331eb0167edf5cd1a
Parents: 5ab149c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Sep 20 23:40:06 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Sep 20 23:40:06 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 2 +-
app-packages/accumulo/pom.xml | 1 +
.../command-logger/slider-pkg/appConfig.json | 2 +-
app-packages/hbase-win/appConfig.json | 2 +-
app-packages/hbase/appConfig.json | 2 +-
app-packages/memcached-win/appConfig.json | 2 +-
app-packages/memcached/appConfig.json | 2 +-
app-packages/storm/appConfig.json | 2 +-
.../test_command_log/appConfig.json | 4 +--
.../test_command_log/appConfig_fast_no_reg.json | 3 +-
.../test_command_log/appConfig_no_hb.json | 4 +--
slider-funtest/pom.xml | 1 +
.../framework/AgentCommandTestBase.groovy | 31 +++++++-------------
.../funtest/lifecycle/AMFailuresIT.groovy | 1 -
.../lifecycle/AgentClusterLifecycleIT.groovy | 1 -
.../funtest/lifecycle/AgentFailures2IT.groovy | 1 -
.../funtest/lifecycle/AgentFailuresIT.groovy | 1 -
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 2 +-
.../lifecycle/ClusterBuildDestroyIT.groovy | 1 -
19 files changed, 24 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index d6f849c..28fb637 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "${app.package.name}.zip",
+ "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
"java_home": "/usr/lib/jvm/java",
"site.global.app_user": "${app.user}",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index c7d007e..02a0a72 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -190,6 +190,7 @@
<slider.bin.dir>${slider.bin.dir}</slider.bin.dir>
<test.app.pkg.dir>${test.app.pkg.dir}</test.app.pkg.dir>
<test.app.pkg.file>${app.package.name}.zip</test.app.pkg.file>
+ <test.app.pkg.name>ACCUMULO</test.app.pkg.name>
<test.app.resources.dir>${test.app.resources.dir}</test.app.resources.dir>
</systemPropertyVariables>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/command-logger/slider-pkg/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json b/app-packages/command-logger/slider-pkg/appConfig.json
index 791043f..73bc7fa 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
"site.global.application_id": "CommandLogger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/hbase-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/appConfig.json b/app-packages/hbase-win/appConfig.json
index 8261f8c..94f1343 100644
--- a/app-packages/hbase-win/appConfig.json
+++ b/app-packages/hbase-win/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "${app.package.name}.zip",
+ "application.def": ".slider/package/HBASE/${app.package.name}.zip",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.7.0_45",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 3743602..78479db 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "slider-hbase-app-package-${bld.version}.zip",
+ "application.def": ".slider/package/HBASE/slider-hbase-app-package-${bld.version}.zip",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"system_configs": "core-site",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
index fa4050b..7863579 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "jmemcached-1.0.0.zip",
+ "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
"java_home": "C:\\java",
"site.global.app_user": "hadoop",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index c335959..cf4a838 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "jmemcached-1.0.0.zip",
+ "application.def": ".slider/package/MEMCACHED/jmemcached-1.0.0.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index 4af092d..9a926c2 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "slider-storm-app-package-${pkg.version}.zip",
+ "application.def": ".slider/package/STORM/slider-storm-app-package-${pkg.version}.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"create.default.zookeeper.node": "true",
"system_configs": "core-site",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-core/src/test/app_packages/test_command_log/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig.json b/slider-core/src/test/app_packages/test_command_log/appConfig.json
index 7586a0c..8e81421 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig.json
@@ -3,15 +3,13 @@
"metadata": {
},
"global": {
- "agent.conf": "agent.ini",
- "application.def": "apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
"site.global.application_id": "CommandLogger",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
"site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
index cbe28de..76858e7 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
@@ -5,8 +5,7 @@
"global": {
"heartbeat.monitor.interval": "20000",
"agent.instance.debug.data": "ANY:DO_NOT_REGISTER:NONE",
- "agent.conf": "agent.ini",
- "application.def": "apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
"site.global.application_id": "CommandLogger",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
index 41c4709..49ec4b7 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
@@ -5,15 +5,13 @@
"global": {
"heartbeat.monitor.interval": "20000",
"agent.instance.debug.data": "ANY:DO_NOT_HEARTBEAT:DO_NOT_HEARTBEAT:NONE",
- "agent.conf": "agent.ini",
- "application.def": "apache-slider-command-logger.zip",
+ "application.def": ".slider/package/CMD_LOGGER/apache-slider-command-logger.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"site.global.app_user": "yarn",
"site.global.application_id": "CommandLogger",
"site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
"site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/pom.xml
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 55b6351..295b8eb 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -119,6 +119,7 @@
<slider.bin.dir>../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
<test.app.pkg.dir>../app-packages/command-logger/slider-pkg/target</test.app.pkg.dir>
<test.app.pkg.file>apache-slider-command-logger.zip</test.app.pkg.file>
+ <test.app.pkg.name>CMD_LOGGER</test.app.pkg.name>
<test.app.resource>../slider-core/src/test/app_packages/test_command_log/resources.json</test.app.resource>
<test.app.template>../slider-core/src/test/app_packages/test_command_log/appConfig.json</test.app.template>
<vagrant.current.working.dir>/Users/gsaha/work/workspace/ambari-vagrant/centos6.4</vagrant.current.working.dir>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index b565cd1..f1edff4 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -37,6 +37,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
public static final boolean AGENTTESTS_ENABLED
private static String TEST_APP_PKG_DIR_PROP = "test.app.pkg.dir"
private static String TEST_APP_PKG_FILE_PROP = "test.app.pkg.file"
+ private static String TEST_APP_PKG_NAME_PROP = "test.app.pkg.name"
private static String TEST_APP_RESOURCE = "test.app.resource"
private static String TEST_APP_TEMPLATE = "test.app.template"
@@ -45,6 +46,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
protected String APP_TEMPLATE = getAppTemplate()
public static final String TEST_APP_PKG_DIR = sysprop(TEST_APP_PKG_DIR_PROP)
public static final String TEST_APP_PKG_FILE = sysprop(TEST_APP_PKG_FILE_PROP)
+ public static final String TEST_APP_PKG_NAME = sysprop(TEST_APP_PKG_NAME_PROP)
protected static Path agentTarballPath;
@@ -79,29 +81,18 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
@Before
- public void uploadAgentTarball() {
- def agentUploads = new AgentUploads(SLIDER_CONFIG)
- (agentTarballPath, agtIniPath) =
- agentUploads.uploadAgentFiles(SLIDER_TAR_DIRECTORY, false)
- }
-
-
- @Before
public void setupApplicationPackage() {
try {
- AgentUploads agentUploads = new AgentUploads(SLIDER_CONFIG)
- agentUploads.uploader.mkHomeDir()
-
- appPkgPath = new Path(clusterFS.homeDirectory, TEST_APP_PKG_FILE)
- if (clusterFS.exists(appPkgPath)) {
- clusterFS.delete(appPkgPath, false)
- log.info "Existing app pkg deleted from $appPkgPath"
- }
-
File zipFileName = new File(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE).canonicalFile
- agentUploads.uploader.copyIfOutOfDate(zipFileName, appPkgPath, false)
- assert clusterFS.exists(appPkgPath), "App pkg not uploaded to $appPkgPath"
- log.info "App pkg uploaded at $appPkgPath"
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, TEST_APP_PKG_NAME,
+ Arguments.ARG_PACKAGE, zipFileName,
+ Arguments.ARG_REPLACE_PKG
+ ])
+ logShell(shell)
+ log.info "App pkg uploaded at home directory .slider/package/$TEST_APP_PKG_NAME/$TEST_APP_PKG_FILE"
} catch (Exception e) {
setup_failed = true
throw e;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
index b92207c..852121f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -57,7 +57,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index 51f1e94..73ca275 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -68,7 +68,6 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, CLUSTER,
- ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE2
])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 0ba48ba..9359521 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -56,7 +56,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE3,
ARG_RESOURCES, APP_RESOURCE
])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index a51c769..fcbfb3c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -56,7 +56,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE2,
ARG_RESOURCES, APP_RESOURCE
])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 00a876a..8930b8d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -50,7 +50,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
SliderShell shell = slider(EXIT_SUCCESS,
[
ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
+ //ARG_IMAGE, agentTarballPath.toString(),
ARG_TEMPLATE, APP_TEMPLATE,
ARG_RESOURCES, APP_RESOURCE
])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0d41051a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index f8caac5..c5dbecb 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -61,7 +61,6 @@ public class ClusterBuildDestroyIT extends AgentCommandTestBase
[
ACTION_BUILD,
CLUSTER,
- ARG_IMAGE, agentTarballPath.toString(),
ARG_ZKHOSTS,
SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
ARG_TEMPLATE, APP_TEMPLATE,
[10/22] git commit: SLIDER-451. slider install-package could use
better error messages
Posted by sm...@apache.org.
SLIDER-451. slider install-package could use better error messages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/47b4a9c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/47b4a9c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/47b4a9c6
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 47b4a9c606d6e7064c690f60ab9f456d8ec6ac75
Parents: 94df22c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Sep 23 21:07:55 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Sep 23 21:07:55 2014 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 6 +++---
.../slider/agent/actions/TestActionInstallPackage.groovy | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47b4a9c6/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 221fc43..129f851 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -623,11 +623,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path srcFile = null;
if (null == installPkgInfo.name || installPkgInfo.name.length() == 0) {
- throw new BadCommandArgumentsException("A valid application name is required.");
+ throw new BadCommandArgumentsException("A valid application type name is required (e.g. HBASE).");
}
if (null == installPkgInfo.packageURI || installPkgInfo.packageURI.length() == 0) {
- throw new BadCommandArgumentsException("A valid application package is required.");
+ throw new BadCommandArgumentsException("A valid application package location required.");
} else {
File pkgFile = new File(installPkgInfo.packageURI);
if (!pkgFile.exists() || pkgFile.isDirectory()) {
@@ -646,7 +646,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
throw new BadCommandArgumentsException("Pkg exists at " +
fileInFs.toUri().toString() +
- ". Use --replacePkg to overwrite.");
+ ". Use --replacepkg to overwrite.");
}
sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47b4a9c6/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
index 88f1298..064c7a9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
@@ -56,7 +56,7 @@ class TestActionInstallPackage extends AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " + launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("A valid application name is required")
+ assert e.message.contains("A valid application type name is required (e.g. HBASE)")
}
}
@@ -74,7 +74,7 @@ class TestActionInstallPackage extends AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " + launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("A valid application package is required.")
+ assert e.message.contains("A valid application package location required")
}
}
@@ -103,7 +103,7 @@ class TestActionInstallPackage extends AgentMiniClusterTestBase {
)
fail("expected an exception, got a status code " + launcher.serviceExitCode)
} catch (BadCommandArgumentsException e) {
- assert e.message.contains("Use --replacePkg to overwrite")
+ assert e.message.contains("Use --replacepkg to overwrite")
}
}
[12/22] git commit: SLIDER-343. Command files have incorrect hostname
Posted by sm...@apache.org.
SLIDER-343. Command files have incorrect hostname
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/314c3cce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/314c3cce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/314c3cce
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 314c3cce5d52e11e04e3d591f1795d128e9b722a
Parents: 9ce88ce
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Sep 24 13:43:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Sep 24 13:43:13 2014 -0700
----------------------------------------------------------------------
.../src/main/python/agent/CustomServiceOrchestrator.py | 7 +++++--
.../test/python/agent/TestCustomServiceOrchestrator.py | 10 ++++++++++
.../server/appmaster/web/rest/agent/ExecutionCommand.java | 1 +
3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/314c3cce/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 30eedde..7b49a67 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -216,8 +216,11 @@ class CustomServiceOrchestrator():
"""
# Perform few modifications to stay compatible with the way in which
# site.pp files are generated by manifestGenerator.py
- public_fqdn = self.public_fqdn
- command['public_hostname'] = public_fqdn
+ command['public_hostname'] = self.public_fqdn
+ if 'hostname' in command:
+ command['appmaster_hostname'] = command['hostname']
+ command['hostname'] = self.public_fqdn
+
# Now, dump the json file
command_type = command['commandType']
from ActionQueue import ActionQueue # To avoid cyclic dependency
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/314c3cce/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 4548aac..2fb1611 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -55,6 +55,7 @@ class TestCustomServiceOrchestrator(TestCase):
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
+ 'hostname' : 'host1',
'componentName': 'NAMENODE',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
@@ -80,6 +81,7 @@ class TestCustomServiceOrchestrator(TestCase):
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
+ self.assertEquals(command['hostname'], "host1")
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
@@ -87,6 +89,12 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
+
+ # Testing side effect of dump_command_to_json
+ self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertEquals(command['hostname'], "test.hst")
+ self.assertEquals(command['appmaster_hostname'], "host1")
+
# Test dumping STATUS_COMMAND
command['commandType'] = 'STATUS_COMMAND'
json_file = orchestrator.dump_command_to_json(command, {})
@@ -97,6 +105,8 @@ class TestCustomServiceOrchestrator(TestCase):
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
+ self.assertEquals(command['hostname'], "test.hst")
+ self.assertEquals(command['appmaster_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/314c3cce/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index 94aa29f..5fb3b5e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -36,6 +36,7 @@ public class ExecutionCommand {
private String clusterName;
private long taskId;
private String commandId;
+ //TODO Remove hostname from being set in the command
private String hostname;
private String role;
private Map<String, String> hostLevelParams = new HashMap<String, String>();
[11/22] git commit: SLIDER-441. Rename DO_NOT_PROPAGATE to be more
user-friendly
Posted by sm...@apache.org.
SLIDER-441. Rename DO_NOT_PROPAGATE to be more user-friendly
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9ce88ced
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9ce88ced
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9ce88ced
Branch: refs/heads/feature/SLIDER-158_slider_diagnostic_option
Commit: 9ce88ced4fe24c5284e110877200367d24248430
Parents: 47b4a9c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Sep 23 21:34:00 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Sep 23 21:34:00 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/appConfig.json | 2 +-
app-packages/memcached/appConfig.json | 2 +-
app-packages/storm/appConfig.json | 4 ++--
.../src/main/python/agent/CustomServiceOrchestrator.py | 8 ++++----
.../test/python/agent/TestCustomServiceOrchestrator.py | 10 +++++-----
.../slider/providers/agent/AgentProviderService.java | 4 ++--
.../slider/providers/agent/TestAgentProviderService.java | 2 +-
7 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
index 7863579..059c712 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -13,7 +13,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index cf4a838..7b4aa97 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -14,7 +14,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{PER_CONTAINER}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index 9a926c2..683b8cc 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -33,10 +33,10 @@
"site.storm-site.transactional.zookeeper.port": "null",
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
- "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER},${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}]",
"site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
- "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{PER_CONTAINER}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index dd8e9b9..30eedde 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -246,7 +246,7 @@ class CustomServiceOrchestrator():
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
ALLOCATED_PORT is a hint to allocate port. It works as follows:
- Its of the form {component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{DO_NOT_PROPAGATE}]
+ Its of the form {component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{PER_CONTAINER}]
Either a port gets allocated or if not then just set the value to "0"
"""
@@ -289,7 +289,7 @@ class CustomServiceOrchestrator():
All unallocated ports should be set to 0
Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}"
or "${component.ALLOCATED_PORT}{DEFAULT_port}"
- or "${component.ALLOCATED_PORT}{DEFAULT_port}{DO_NOT_PROPAGATE}"
+ or "${component.ALLOCATED_PORT}{DEFAULT_port}{PER_CONTAINER}"
"""
def set_all_unallocated_ports(self, value):
@@ -318,11 +318,11 @@ class CustomServiceOrchestrator():
Port allocation can asks for multiple dynamic ports
port_req_pattern is of type ${component_name.ALLOCATED_PORT}
append {DEFAULT_ and find the default value
- append {DO_NOT_PROPAGATE} if it exists
+ append {PER_CONTAINER} if it exists
"""
def allocate_ports(self, value, port_req_pattern):
default_port_pattern = "{DEFAULT_"
- do_not_propagate_pattern = "{DO_NOT_PROPAGATE}"
+ do_not_propagate_pattern = "{PER_CONTAINER}"
index = value.find(port_req_pattern)
while index != -1:
replaced_pattern = port_req_pattern
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index e545afe..4548aac 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -310,9 +310,9 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret, "102,103")
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "104")
- ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}{PER_CONTAINER}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "105")
- ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{PER_CONTAINER}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "106")
@@ -336,7 +336,7 @@ class TestCustomServiceOrchestrator(TestCase):
"${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1005-1006")
- ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1006}{DO_NOT_PROPAGATE}",
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1006}{PER_CONTAINER}",
"${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1006")
@@ -489,8 +489,8 @@ class TestCustomServiceOrchestrator(TestCase):
command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
command['configurations']['oozie-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
command['configurations']['oozie-site']['ignore_port1'] = "[${HBASE_RS.ALLOCATED_PORT}]"
- command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]"
- command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${A.ALLOCATED_PORT}{DEFAULT_3}{DO_NOT_PROPAGATE}]"
+ command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{PER_CONTAINER}]"
+ command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{PER_CONTAINER},${A.ALLOCATED_PORT}{DEFAULT_3}{PER_CONTAINER}]"
command['configurations']['oozie-site']['ignore_port4'] = "${HBASE_RS}{a}{b}{c}"
allocated_ports = {}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 6976f50..f37150e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -126,7 +126,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
private static final String SHARED_PORT_TAG = "SHARED";
- private static final String DO_NOT_PROPAGATE_TAG = "{DO_NOT_PROPAGATE}";
+ private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
@@ -1482,7 +1482,7 @@ public class AgentProviderService extends AbstractProviderService implements
for (String key : config.keySet()) {
String value = config.get(key);
String lookupKey = configName + "." + key;
- if (!value.contains(DO_NOT_PROPAGATE_TAG)) {
+ if (!value.contains(PER_CONTAINER_TAG)) {
// If the config property is shared then pass on the already allocated value
// from any container
if (this.getAllocatedPorts().containsKey(lookupKey)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9ce88ced/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index b652bb5..10b20d0 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -1268,7 +1268,7 @@ public class TestAgentProviderService {
treeOps.set(OptionKeys.ZOOKEEPER_HOSTS, "HOST1");
treeOps.getGlobalOptions().put("site.hbase-site.a.port", "${HBASE_MASTER.ALLOCATED_PORT}");
treeOps.getGlobalOptions().put("site.hbase-site.b.port", "${HBASE_MASTER.ALLOCATED_PORT}");
- treeOps.getGlobalOptions().put("site.hbase-site.random.port", "${HBASE_MASTER.ALLOCATED_PORT}{DO_NOT_PROPAGATE}");
+ treeOps.getGlobalOptions().put("site.hbase-site.random.port", "${HBASE_MASTER.ALLOCATED_PORT}{PER_CONTAINER}");
treeOps.getGlobalOptions().put("site.hbase-site.random2.port", "${HBASE_MASTER.ALLOCATED_PORT}");
Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>();