You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/06/30 17:36:50 UTC
[01/50] [abbrv] git commit: SLIDER-153: stub .py command
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it 6f5c5faf1 -> 81a85495d
SLIDER-153: stub .py command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/19d313d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/19d313d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/19d313d7
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 19d313d7fb879ff8aff24494800870a704100df1
Parents: 5023eb5
Author: Steve Loughran <st...@apache.org>
Authored: Fri Jun 20 18:07:54 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Jun 20 18:07:54 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/19d313d7/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
new file mode 100644
index 0000000..f154210
--- /dev/null
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -0,0 +1,22 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Slider main method
+
+if __name__ == '__main__':
+ print "slider python script"
\ No newline at end of file
[36/50] [abbrv] git commit: SLIDER-153 revert rename -> slider for
bash
Posted by st...@apache.org.
SLIDER-153 revert rename -> slider for bash
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/150b589b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/150b589b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/150b589b
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 150b589b72dae4c4b857a6b19205f6ce11f63d39
Parents: 1c5f303
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jun 25 11:22:20 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jun 25 11:22:20 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 74 +++++++++++++++++++++++++
slider-assembly/src/main/scripts/slider.sh | 74 -------------------------
2 files changed, 74 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/150b589b/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
new file mode 100755
index 0000000..caf275b
--- /dev/null
+++ b/slider-assembly/src/main/scripts/slider
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# this is the shell script to start Slider deploying an application
+# Usage: slider <action> <commands>
+
+# The env variable SLIDER_JVM_OPTS can be used to override
+# the default JVM opts
+
+function usage
+{
+ echo "Usage: slider <action> <arguments>"
+ echo ""
+}
+
+# Slider works out its own location
+this="${BASH_SOURCE-$0}"
+bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
+script="$(basename -- "$this")"
+
+# lib directory is one up; it is expected to contain
+# slider.jar and any other dependencies that are not in the
+# standard Hadoop classpath
+
+slider_home="${bindir}/.."
+slider_home=`cd -P "${slider_home}" && pwd -P`
+
+libdir="${slider_home}/lib"
+libdir=`cd -P "${libdir}" && pwd -P`
+
+
+confdir="${slider_home}/conf"
+
+# normalize the conf dir so it can be passed down
+confdir=`cd -P "${confdir}" && pwd -P`
+confdir=${SLIDER_CONF_DIR:-$confdir}
+
+
+slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}"
+slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
+
+# allow for an extra classpath
+slider_classpath_extra=${SLIDER_CLASSPATH_EXTRA:-""}
+
+slider_classpath="${libdir}/*:${confdir}:${slider_classpath_extra}"
+
+launcher=org.apache.slider.Slider
+
+
+echo "slider_home = \"${slider_home}\""
+echo "slider_jvm_opts = \"${slider_jvm_opts}\""
+echo "classpath = \"${slider_classpath}\""
+export CLASSPATH="${slider_classpath}"
+echo ""
+
+echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\" ${launcher} $@"
+echo ""
+echo ""
+exec java ${slider_jvm_opts} ${launcher} $@
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/150b589b/slider-assembly/src/main/scripts/slider.sh
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.sh b/slider-assembly/src/main/scripts/slider.sh
deleted file mode 100755
index caf275b..0000000
--- a/slider-assembly/src/main/scripts/slider.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# this is the shell script to start Slider deploying an application
-# Usage: slider <action> <commands>
-
-# The env variable SLIDER_JVM_OPTS can be used to override
-# the default JVM opts
-
-function usage
-{
- echo "Usage: slider <action> <arguments>"
- echo ""
-}
-
-# Slider works out its own location
-this="${BASH_SOURCE-$0}"
-bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
-
-# lib directory is one up; it is expected to contain
-# slider.jar and any other dependencies that are not in the
-# standard Hadoop classpath
-
-slider_home="${bindir}/.."
-slider_home=`cd -P "${slider_home}" && pwd -P`
-
-libdir="${slider_home}/lib"
-libdir=`cd -P "${libdir}" && pwd -P`
-
-
-confdir="${slider_home}/conf"
-
-# normalize the conf dir so it can be passed down
-confdir=`cd -P "${confdir}" && pwd -P`
-confdir=${SLIDER_CONF_DIR:-$confdir}
-
-
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}"
-slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
-
-# allow for an extra classpath
-slider_classpath_extra=${SLIDER_CLASSPATH_EXTRA:-""}
-
-slider_classpath="${libdir}/*:${confdir}:${slider_classpath_extra}"
-
-launcher=org.apache.slider.Slider
-
-
-echo "slider_home = \"${slider_home}\""
-echo "slider_jvm_opts = \"${slider_jvm_opts}\""
-echo "classpath = \"${slider_classpath}\""
-export CLASSPATH="${slider_classpath}"
-echo ""
-
-echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\" ${launcher} $@"
-echo ""
-echo ""
-exec java ${slider_jvm_opts} ${launcher} $@
[24/50] [abbrv] git commit: SLIDER-121 removed site documentation
from git source
Posted by st...@apache.org.
SLIDER-121 removed site documentation from git source
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/209cee43
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/209cee43
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/209cee43
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 209cee43355b943dd55f368ee55a5234ebc3e54b
Parents: 83e4e47
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Jun 24 12:53:38 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue Jun 24 12:53:38 2014 -0700
----------------------------------------------------------------------
pom.xml | 136 ---
src/site/markdown/architecture/architecture.md | 142 ---
src/site/markdown/architecture/index.md | 27 -
src/site/markdown/architecture/rolehistory.md | 1010 ------------------
src/site/markdown/client-configuration.md | 310 ------
src/site/markdown/configuration/core.md | 407 -------
.../example-app_configuration-resolved.json | 42 -
.../example-app_configuration.json | 25 -
.../markdown/configuration/example-empty.json | 8 -
.../configuration/example-internal.json | 21 -
.../example-overridden-resolved.json | 25 -
.../configuration/example-overridden.json | 23 -
.../configuration/example-resources.json | 25 -
.../markdown/configuration/index-markdown.md | 31 -
src/site/markdown/configuration/index.md | 38 -
.../markdown/configuration/original-hbase.json | 139 ---
.../markdown/configuration/proposed-hbase.json | 273 -----
src/site/markdown/configuration/redesign.md | 478 ---------
.../configuration/resolved-resources.json | 22 -
.../markdown/configuration/specification.md | 512 ---------
src/site/markdown/debugging.md | 92 --
src/site/markdown/developing/building.md | 374 -------
.../markdown/developing/functional_tests.md | 416 --------
src/site/markdown/developing/index.md | 35 -
src/site/markdown/developing/manual_testing.md | 53 -
src/site/markdown/developing/releasing.md | 195 ----
src/site/markdown/developing/testing.md | 182 ----
src/site/markdown/examples.md | 159 ---
src/site/markdown/exitcodes.md | 161 ---
src/site/markdown/getting_started.md | 509 ---------
src/site/markdown/index.md | 94 --
src/site/markdown/manpage.md | 483 ---------
.../registry/a_YARN_service_registry.md | 226 ----
src/site/markdown/registry/index.md | 47 -
.../registry/initial_registry_design.md | 110 --
.../markdown/registry/p2p_service_registries.md | 137 ---
src/site/markdown/registry/references.md | 49 -
src/site/markdown/registry/registry-model.md | 75 --
.../service_registry_end_to_end_scenario.md | 156 ---
...lication_registration_and_binding_problem.md | 192 ----
.../markdown/release_notes/release-0.30.0.md | 36 -
src/site/markdown/security.md | 197 ----
.../slider_specs/application_configuration.md | 82 --
.../slider_specs/application_definition.md | 182 ----
.../application_instance_configuration.md | 104 --
.../markdown/slider_specs/application_needs.md | 140 ---
.../slider_specs/application_package.md | 147 ---
.../slider_specs/canonical_scenarios.md | 165 ---
.../slider_specs/creating_app_definitions.md | 135 ---
src/site/markdown/slider_specs/index.md | 53 -
.../slider_specs/resource_specification.md | 53 -
.../slider_specs/writing_app_command_scripts.md | 211 ----
src/site/markdown/specification/cli-actions.md | 675 ------------
src/site/markdown/specification/index.md | 41 -
src/site/markdown/specification/slider-model.md | 286 -----
src/site/markdown/troubleshooting.md | 154 ---
src/site/resources/hoya_am_architecture.png | Bin 137154 -> 0 bytes
.../resources/images/app_config_folders_01.png | Bin 21050 -> 0 bytes
.../resources/images/app_package_sample_04.png | Bin 67927 -> 0 bytes
src/site/resources/images/image_0.png | Bin 194914 -> 0 bytes
src/site/resources/images/image_1.png | Bin 77882 -> 0 bytes
src/site/resources/images/managed_client.png | Bin 71352 -> 0 bytes
src/site/resources/images/slider-container.png | Bin 70265 -> 0 bytes
src/site/resources/images/unmanaged_client.png | Bin 58813 -> 0 bytes
src/site/site.xml | 84 --
65 files changed, 10184 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a1d3346..e517063 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,142 +63,6 @@
</developerConnection>
</scm>
- <developers>
- <developer>
- <id>stevel</id>
- <name>Steve Loughran</name>
- <roles>
- <role>PPMC Chair</role>
- </roles>
- <timezone>0</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>billie</id>
- <name>Billie Rinaldi</name>
- <roles>
- <role>PPMC Member</role>
- </roles>
- <timezone>-5</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>ddas</id>
- <name>Devaraj Das</name>
- <roles>
- <role>PPMC Member</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>tedyu</id>
- <name>Ted Yu</name>
- <roles>
- <role>PPMC Member</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>elserj</id>
- <name>Josh Elser</name>
- <roles>
- <role>PPMC Member</role>
- </roles>
- <timezone>-5</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>smohanty</id>
- <name>Sumit Mohanty</name>
- <roles>
- <role>PPMC Member</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>jmaron</id>
- <name>Jon Maron</name>
- <roles>
- <role>PPMC Member</role>
- </roles>
- <timezone>-5</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>vinodkv</id>
- <name>Vinod Kumar Vavilapalli</name>
- <roles>
- <role>Mentor</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>jbonofre</id>
- <name>Jean-Baptiste Onofré</name>
- <roles>
- <role>Mentor</role>
- </roles>
- <timezone></timezone>
- <organization>Talend</organization>
- <organizationUrl>http://www.talend.com</organizationUrl>
- </developer>
-
- <developer>
- <id>mahadev</id>
- <name>Mahadev Konar</name>
- <roles>
- <role>Mentor</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>acmurthy</id>
- <name>Arun Murthy</name>
- <roles>
- <role>Mentor</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
- <developer>
- <id>ddas</id>
- <name>Devaraj Das</name>
- <roles>
- <role>Mentor</role>
- </roles>
- <timezone>-8</timezone>
- <organization>Hortonworks</organization>
- <organizationUrl>http://www.hortonworks.com</organizationUrl>
- </developer>
-
-
- </developers>
-
<distributionManagement>
<site>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/architecture/architecture.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/architecture/architecture.md b/src/site/markdown/architecture/architecture.md
deleted file mode 100644
index c54d94d..0000000
--- a/src/site/markdown/architecture/architecture.md
+++ /dev/null
@@ -1,142 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Architecture
-
-## Summary
-
-Slider is a YARN application to deploy non-YARN-enabled applications in a YARN cluster
-
-Slider consists of a YARN application master, the "Slider AM", and a client
-application which communicates with YARN and the Slider AM via remote procedure
-calls and/or REST requests. The client application offers command line access
- as well as low-level API access for test purposes
-
-The deployed application must be a program that can be run across a pool of
-YARN-managed servers, dynamically locating its peers. It is not Slider's
-responsibility to configure up the peer servers, apart from some initial
-application-specific application instance configuration. (The full requirements
-of an application are [described in another document](app_needs.md).
-
-Every application instance is described as a set of one or more *component*; each
-component can have a different program/command, and a different set of configuration
-options and parameters.
-
-The AM takes the details on which roles to start, and requests a YARN container
-for each component; It then monitors the state of the application instance, receiving messages
-from YARN when a remotely executed process finishes. It then deploys another instance of
-that component.
-
-
-## Slider Packaging
-
-A key goal of Slider is to support the deployment of existing applications into
-a YARN application instance, without having to extend Slider itself.
-
-
-
-## AM Architecture
-
-The application master consists of
-
- 1. The AM engine which handles all integration with external services, specifically YARN and any Slider clients
- 1. A *provider* specific to deploying a class of applications.
- 1. The Application State.
-
-The Application State is the model of the application instance, containing
-
- 1. A specification of the desired state of the application instance -the number of instances of each role, their YARN and process memory requirements and some other options.
- 1. A map of the current instances of each role across the YARN cluster, including reliability statistics of each node in the application instance used.
- 1. [The Role History](rolehistory.html) -a record of which nodes roles were deployed on for re-requesting the same nodes in future. This is persisted to disk and re-read if present, for faster application startup times.
- 1. Queues of track outstanding requests, released and starting nodes
-
-The Application Engine integrates with the outside world: the YARN Resource Manager ("the RM"), and the node-specific Node Managers, receiving events from the services, requesting or releasing containers via the RM, and starting applications on assigned containers.
-
-After any notification of a change in the state of the cluster (or an update to the client-supplied cluster specification), the Application Engine passes the information on to the Application State class, which updates its state and then returns a list of cluster operations to be submitted: requests for containers of different types -potentially on specified nodes, or requests to release containers.
-
-As those requests are met and allocation messages passed to the Application Engine, it works with the Application State to assign them to specific components, then invokes the provider to build up the launch context for that application.
-
-The provider has the task of populating container requests with the file references, environment variables and commands needed to start the provider's supported programs.
-
-The core provider deploys a minimal agent on the target containers, then, as the agent checks in to the agent provider's REST API, executes commands issued to it.
-
-The set of commands this agent executes focuses on downloading archives from HDFS, expanding them, then running Python scripts which perform the
-actual configuration and execution of the target problem -primarily through template expansion.
-
-
-To summarize: Slider is not an classic YARN analysis application, which allocates and schedules work across the cluster in short-to-medium life containers with the lifespan of a query or an analytics session, but instead for an application with a lifespan of days to months. Slider works to keep the actual state of its application cluster to match the desired state, while the application has the tasks of recovering from node failure, locating peer nodes and working with data in an HDFS filesystem.
-
-As such it is one of the first applications designed to use YARN as a platform for long-lived services -Samza being the other key example. These application's needs of YARN are different, and their application manager design is focused around maintaining the distributed application in its desired state rather than the ongoing progress of submitted work.
-
-The clean model-view-controller split was implemented to isolate the model and aid mock testing of large clusters with simulated scale, and hence increase confidence that Slider can scale to work in large YARN clusters and with larger application instances.
-
-
-
-### Failure Model
-
-The application master is designed to be a [crash-only application](https://www.usenix.org/legacy/events/hotos03/tech/full_papers/candea/candea.pdf), clients are free to terminate
-the application instance by asking YARN directly.
-
-There is an RPC call to stop the application instance - this is a nicety which includes a message in the termination log, and
-could, in future, perhaps warn the provider that the application instance is being torn down. That is a potentially dangerous feature
-to add -as provider implementors may start to expect the method to be called reliably. Slider is designed to fail without
-warning, to rebuild its state on a YARN-initiated restart, and to be manually terminated without any advance notice.
-
-### RPC Interface
-
-
-The RPC interface allows the client to query the current application state, and to update it by pushing out a new JSON specification.
-
-The core operations are
-
-* `getJSONClusterStatus()`: get the status of the application instance as a JSON document.
-* `flexCluster()` update the desired count of role instances in the running application instance.
-* `stopCluster` stop the application instance
-
-There are some other low-level operations for extra diagnostics and testing, but they are of limited importancs
-
-The `flexCluster()` call takes a JSON application instance specification and forwards it to the AM -which extracts the desired counts of each role to update the Application State. A change in the desired size of the application instance, is treated as any reported failure of node:
-it triggers a re-evaluation of the application state, building up the list of container add and release requests to make of
-the YARN resource manager.
-
-The final operation, `stopCluster()`, stops the application instance.
-
-### Security and Identity
-
-Slider's security model is described in detail in [an accompanying document](../security.html)
-
-A Slider application instance is expected to access data belonging to the user creating the instance.
-
-In a secure YARN cluster, this is done by acquiring Kerberos tokens in the client when the application instance is updated, tokens which
-are propagated to the Slider AM and thence to the deployed application containers themselves. These
-tokens are valid for a finite time period.
-
-HBase has always required keytab files to be installed on every node in the Hadoop for it to have secure access -this requirement
-holds for Slider-deployed HBase clusters. Slider does not itself adopt the responsibility of preparing or distributing these files;
-this must be done via another channel.
-
-In Hadoop 2.2, the tokens for communication between the Slider AM and YARN expire after -by default- 72 hours. The
-HDFS tokens will also expire after some time period. This places an upper bound on the lifespan of a Slider application (or any
-other long-lived YARN application) in a secure Hadoop cluster.
-
-
-
-In an insecure Hadoopp cluster, the Slider AM and its containers are likely to run in a different OS account from the submitting user.
-To enable access to the database files as that submitting use, the identity of the user is provided when the AM is created; the
-AM will pass this same identity down to the created containers. This information *identifies* the user -but does not *authenticate* them: they are trusted to be who they claim to be.
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/architecture/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/architecture/index.md b/src/site/markdown/architecture/index.md
deleted file mode 100644
index 96fc99c..0000000
--- a/src/site/markdown/architecture/index.md
+++ /dev/null
@@ -1,27 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Architecture
-
-* [Overview](architecture.html)
-* [Application Needs](../slider_specs/application_needs.html)
-* [Specification](../specification/index.html)
-* [Service Registry](../registry/index.html)
-* [Role history](rolehistory.html)
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/architecture/rolehistory.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/architecture/rolehistory.md b/src/site/markdown/architecture/rolehistory.md
deleted file mode 100644
index 497f067..0000000
--- a/src/site/markdown/architecture/rolehistory.md
+++ /dev/null
@@ -1,1010 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Role History: how Slider brings back nodes in the same location
-
-### Last updated 2013-12-06
-
-* This document uses the pre-slider terminology of role/cluster and not
-component and application instance *
-
-
-## Outstanding issues
-
-1. Can we use the history to implement anti-affinity: for any role with this flag,
-use our knowledge of the cluster to ask for all nodes that aren't in use already
-
-1. How to add blacklisting here? We are tracking failures and startup failures
-per node (not persisted), but not using this in role placement requests yet.
-
-## Introduction
-
-Slider needs to bring up instances of a given role on the machine(s) on which
-they last ran -it should remember after shrinking or freezing a cluster which
-servers were last used for a role -and use this (persisted) data to select
-clusters next time
-
-It does this in the basis that the role instances prefer node-local
-access to data previously persisted to HDFS. This is precisely the case
-for Apache HBase, which can use Unix Domain Sockets to talk to the DataNode
-without using the TCP stack. The HBase master persists to HDFS the tables
-assigned to specific Region Servers, and when HBase is restarted its master
-tries to reassign the same tables back to Region Servers on the same machine.
-
-For this to work in a dynamic cluster, Slider needs to bring up Region Servers
-on the previously used hosts, so that the HBase Master can re-assign the same
-tables.
-
-Note that it does not need to care about the placement of other roles, such
-as the HBase masters -there anti-affinity between other instances is
-the key requirement.
-
-### Terminology
-
-* **Role Instance** : a single instance of a role.
-* **Node** : A server in the YARN Physical (or potentially virtual) Cluster of servers.
-* **Slider Cluster**: The set of role instances deployed by Slider so as to
- create a single aggregate application.
-* **Slider AM**: The Application Master of Slider: the program deployed by YARN to
-manage its Slider Cluster.
-* **RM** YARN Resource Manager
-
-### Assumptions
-
-Here are some assumptions in Slider's design
-
-1. Instances of a specific role should preferably be deployed onto different
-servers. This enables Slider to only remember the set of server nodes onto
-which instances were created, rather than more complex facts such as "two Region
-Servers were previously running on Node #17. On restart Slider can simply request
-one instance of a Region Server on a specific node, leaving the other instance
-to be arbitrarily deployed by YARN. This strategy should help reduce the *affinity*
-in the role deployment, so increase their resilience to failure.
-
-1. There is no need to make sophisticated choices on which nodes to request
-re-assignment -such as recording the amount of data persisted by a previous
-instance and prioritizing nodes based on such data. More succinctly 'the
-only priority needed when asking for nodes is *ask for the most recently used*.
-
-1. Different roles are independent: it is not an issue if a role of one type
- (example, an Accumulo Monitor and an Accumulo Tablet Server) are on the same
- host. This assumption allows Slider to only worry about affinity issues within
- a specific role, rather than across all roles.
-
-1. After a cluster has been started, the rate of change of the cluster is
-low: both node failures and cluster flexing happen at the rate of every few
-hours, rather than every few seconds. This allows Slider to avoid needing
-data structures and layout persistence code designed for regular and repeated changes.
-
-1. Instance placement is best-effort: if the previous placement cannot be satisfied,
-the application will still perform adequately with role instances deployed
-onto new servers. More specifically, if a previous server is unavailable
-for hosting a role instance due to lack of capacity or availability, Slider
-will not decrement the number of instances to deploy: instead it will rely
-on YARN to locate a new node -ideally on the same rack.
-
-1. If two instances of the same role do get assigned to the same server, it
-is not a failure condition. (This may be problematic for some roles
--we may need a role-by-role policy here, so that master nodes can be anti-affine)
-[specifically, >1 HBase master mode will not come up on the same host]
-
-1. If a role instance fails on a specific node, asking for a container on
-that same node for the replacement instance is a valid recovery strategy.
-This contains assumptions about failure modes -some randomness here may
-be a valid tactic, especially for roles that do not care about locality.
-
-1. Tracking failure statistics of nodes may be a feature to add in future;
-designing the Role History datastructures to enable future collection
-of rolling statistics on recent failures would be a first step to this
-
-### The Role History
-
-The `RoleHistory` is a datastructure which models the role assignment, and
-can persist it to and restore it from the (shared) filesystem.
-
-* For each role, there is a list of cluster nodes which have supported this role
-used in the past.
-
-* This history is used when selecting a node for a role.
-
-* This history remembers when nodes were allocated. These are re-requested
-when thawing a cluster.
-
-* It must also remember when nodes were released -these are re-requested
-when returning the cluster size to a previous size during flex operations.
-
-* It has to track nodes for which Slider has an outstanding container request
-with YARN. This ensures that the same node is not requested more than once
-due to outstanding requests.
-
-* It does not retain a complete history of the role -and does not need to.
-All it needs to retain is the recent history for every node onto which a role
-instance has been deployed. Specifically, the last allocation or release
-operation on a node is all that needs to be persisted.
-
-* On AM startup, all nodes in the history are considered candidates, even those nodes currently marked
-as active -as they were from the previous instance.
-
-* On AM restart, nodes in the role history marked as active have to be considered
-still active -the YARN RM will have to provide the full list of which are not.
-
-* During cluster flexing, nodes marked as released -and for which there is no
-outstanding request - are considered candidates for requesting new instances.
-
-* When choosing a candidate node for hosting a role instance, it from the head
-of the time-ordered list of nodes that last ran an instance of that role
-
-### Persistence
-
-The state of the role is persisted to HDFS on changes -but not on cluster
-termination.
-
-1. When nodes are allocated, the Role History is marked as dirty
-1. When container release callbacks are received, the Role History is marked as dirty
-1. When nodes are requested or a release request made, the Role History is *not*
- marked as dirty. This information is not relevant on AM restart.
-
-As at startup, a large number of allocations may arrive in a short period of time,
-the Role History may be updated very rapidly -yet as the containers are
-only recently activated, it is not likely that an immediately restarted Slider
-cluster would gain by re-requesting containers on them -their historical
-value is more important than their immediate past.
-
-Accordingly, the role history may be persisted to HDFS asynchronously, with
-the dirty bit triggering an flushing of the state to HDFS. The datastructure
-will still need to be synchronized for cross thread access, but the
-sync operation will not be a major deadlock, compared to saving the file on every
-container allocation response (which will actually be the initial implementation).
-
-There's no need to persist the format in a human-readable form; while protobuf
-might seem the approach most consistent with the rest of YARN, it's not
-an easy structure to work with.
-
-The initial implementation will use Apache Avro as the persistence format,
-with the data saved in JSON or compressed format.
-
-
-## Weaknesses in this design
-
-**Blacklisting**: even if a node fails repeatedly, this design will still try to re-request
-instances on this node; there is no blacklisting. As a central blacklist
-for YARN has been proposed, it is hoped that this issue will be addressed centrally,
-without Slider having to remember which nodes are unreliable *for that particular
-Slider cluster*.
-
-**Anti-affinity**: If multiple role instances are assigned to the same node,
-Slider has to choose on restart or flexing whether to ask for multiple
-nodes on that node again, or to pick other nodes. The assumed policy is
-"only ask for one node"
-
-**Bias towards recent nodes over most-used**: re-requesting the most
-recent nodes, rather than those with the most history of use, may
-push Slider to requesting nodes that were only briefly in use -and so have
-on a small amount of local state, over nodes that have had long-lived instances.
-This is a problem that could perhaps be addressed by preserving more
-history of a node -maintaining some kind of moving average of
-node use and picking the heaviest used, or some other more-complex algorithm.
-This may be possible, but we'd need evidence that the problem existed before
-trying to address it.
-
-# The NodeMap: the core of the Role History
-
-The core data structure, the `NodeMap` is a map of every known node in the cluster, tracking
-how many containers are allocated to specific roles in it, and, when there
-are no active instances, when it was last used. This history is used to
-choose where to request new containers. Because of the asynchronous
-allocation and release of containers, the Role History also needs to track
-outstanding release requests --and, more critically, outstanding allocation
-requests. If Slider has already requested a container for a specific role
-on a host, then asking for another container of that role would break
-anti-affinity requirements. Note that not tracking outstanding requests would
-radically simplify some aspects of the design, especially the complexity
-of correlating allocation responses with the original requests -and so the
-actual hosts originally requested.
-
-1. Slider builds up a map of which nodes have recently been used.
-1. Every node counts the number. of active containers in each role.
-1. Nodes are only chosen for allocation requests when there are no
-active or requested containers on that node.
-1. When choosing which instances to release, Slider could pick the node with the
-most containers on it. This would spread the load.
-1. When there are no empty nodes to request containers on, a request would
-let YARN choose.
-
-#### Strengths
-
-* Handles the multi-container on one node problem
-* By storing details about every role, cross-role decisions could be possible
-* Simple counters can track the state of pending add/release requests
-* Scales well to a rapidly flexing cluster
-* Simple to work with and persist
-* Easy to view and debug
-* Would support cross-role collection of node failures in future
-
-#### Weaknesses
-
-* Size of the data structure is `O(nodes * role-instances`). This
-could be mitigated by regular cleansing of the structure. For example, at
-thaw time (or intermittently) all unused nodes > 2 weeks old could be dropped.
-* Locating a free node could take `O(nodes)` lookups -and if the criteria of "newest"
-is included, will take exactly `O(nodes)` lookups. As an optimization, a list
-of recently explicitly released nodes can be maintained.
-* Need to track outstanding requests against nodes, so that if a request
-was satisfied on a different node, the original node's request count is
- decremented, *not that of the node actually allocated*.
-* In a virtual cluster, may fill with node entries that are no longer in the cluster.
-Slider should query the RM (or topology scripts?) to determine if nodes are still
-parts of the YARN cluster.
-
-## Data Structures
-
-### RoleHistory
-
- startTime: long
- saveTime: long
- dirty: boolean
- nodemap: NodeMap
- roles: RoleStatus[]
- outstandingRequests: transient OutstandingRequestTracker
- availableNodes: transient List<NodeInstance>[]
-
-This is the aggregate data structure that is persisted to/from file
-
-### NodeMap
-
- clusterNodes: Map: NodeId -> NodeInstance
- clusterNodes(): Iterable<NodeInstance>
- getOrCreate(NodeId): NodeInstance
-
- Maps a YARN NodeID record to a Slider `NodeInstance` structure
-
-### NodeInstance
-
-Every node in the cluster is modeled as an ragged array of `NodeEntry` instances, indexed
-by role index -
-
- NodeEntry[roles]
- get(roleId): NodeEntry or null
- create(roleId): NodeEntry
- getNodeEntries(): NodeEntry[roles]
- getOrCreate(roleId): NodeEntry
- remove(roleId): NodeEntry
-
-This could be implemented in a map or an indexed array; the array is more
-efficient but it does mandate that the number of roles are bounded and fixed.
-
-### NodeEntry
-
-Records the details about all of a roles containers on a node. The
-`active` field records the number of containers currently active.
-
- active: int
- requested: transient int
- releasing: transient int
- last_used: long
-
- NodeEntry.available(): boolean = active - releasing == 0 && requested == 0
-
-The two fields `releasing` and `requested` are used to track the ongoing
-state of YARN requests; they do not need to be persisted across freeze/thaw
-cycles. They may be relevant across AM restart, but without other data
-structures in the AM, not enough to track what the AM was up to before
-it was restarted. The strategy will be to ignore unexpected allocation
-responses (which may come from pre-restart) requests, while treating
-unexpected container release responses as failures.
-
-The `active` counter is only decremented after a container release response
-has been received.
-
-### RoleStatus
-
-This is the existing `org.apache.hoya.yarn.appmaster.state.RoleStatus` class
-
-### RoleList
-
-A list mapping role to int enum is needed to index NodeEntry elements in
-the NodeInstance arrays. Although such an enum is already implemented in the Slider
-Providers, explicitly serializing and deserializing it would make
-the persistent structure easier to parse in other tools, and resilient
-to changes in the number or position of roles.
-
-This list could also retain information about recently used/released nodes,
-so that the selection of containers to request could shortcut a search
-
-
-### ContainerPriority
-
-The container priority field (a 32 bit integer) is used by Slider (0.5.x)
-to index the specific role in a container so as to determine which role
-has been offered in a container allocation message, and which role has
-been released on a release event.
-
-The Role History needs to track outstanding requests, so that
-when an allocation comes in, it can be mapped back to the original request.
-Simply looking up the nodes on the provided container and decrementing
-its request counter is not going to work -the container may be allocated
-on a different node from that requested.
-
-**Proposal**: The priority field of a request is divided by Slider into 8 bits for
-`roleID` and 24 bits for `requestID`. The request ID will be a simple
-rolling integer -Slider will assume that after 2^24 requests per role, it can be rolled,
--though as we will be retaining a list of outstanding requests, a clash should not occur.
-The main requirement is: not have > 2^24 outstanding requests for instances of a specific role,
-which places an upper bound on the size of a Slider cluster.
-
-The splitting and merging will be implemented in a ContainerPriority class,
-for uniform access.
-
-### OutstandingRequest ###
-
-Tracks an outstanding request. This is used to correlate an allocation response
-(whose Container Priority file is used to locate this request), with the
-node and role used in the request.
-
- roleId: int
- requestID : int
- node: string (may be null)
- requestedTime: long
- priority: int = requestID << 24 | roleId
-
-The node identifier may be null -which indicates that a request was made without
-a specific target node
-
-### OutstandingRequestTracker ###
-
-Contains a map from requestID to the specific `OutstandingRequest` made,
-and generates the request ID
-
- nextRequestId: int
- requestMap(RequestID) -> OutstandingRequest
-
-Operations
-
- addRequest(NodeInstance, RoleId): OutstandingRequest
- (and an updated request Map with a new entry)
- lookup(RequestID): OutstandingRequest
- remove(RequestID): OutstandingRequest
- listRequestsForNode(ClusterID): [OutstandingRequest]
-
-The list operation can be implemented inefficiently unless it is found
-to be important -if so a more complex structure will be needed.
-
-### AvailableNodes
-
-This is a field in `RoleHistory`
-
- availableNodes: List<NodeInstance>[]
-
-
-For each role, lists nodes that are available for data-local allocation,
-ordered by more recently released - To accelerate node selection
-
-The performance benefit is most significant when requesting multiple nodes,
-as the scan for M locations from N nodes is reduced from `M*N` comparisons
-to 1 Sort + M list lookups.
-
-Each list can be created off the Node Map by building for each role a sorted
-list of all Nodes which are available for an instance of that role,
-using a comparator that places the most recently released node ahead of older
-nodes.
-
-This list is not persisted -when a Slider Cluster is frozen it is moot, and when
-an AM is restarted this structure will be rebuilt.
-
-1. When a node is needed for a new request, this list is consulted first.
-1. After the request is issued it can be removed from the list
-1. Whenever a container is released, if the node is now available for
-requests for that node, should be added to to the front
-of the list for that role.
-
-If the list is empty during a container request operation, it means
-that the Role History does not know of any nodes
-in the cluster that have hosted instances of that role and which are not
-in use. There are then two possible strategies to select a role
-
-1. Ask for an instance anywhere in the cluster (policy in Slider 0.5)
-1. Search the node map to identify other nodes which are (now) known about,
-but which are not hosting instances of a specific role -this can be used
-as the target for the next resource request.
-
-Strategy #1 is simpler; Strategy #2 *may* decrease the affinity in the cluster,
-as the AM will be explicitly requesting an instance on a node which it knows
-is not running an instance of that role.
-
-
-#### ISSUE What to do about failing nodes?
-
-Should a node whose container just failed be placed at the
-top of the stack, ready for the next request?
-
-If the container failed due to an unexpected crash in the application, asking
-for that container back *is the absolute right strategy* -it will bring
-back a new role instance on that machine.
-
-If the container failed because the node is now offline, the container request
-will not be satisfied by that node.
-
-If there is a problem with the node, such that containers repeatedly fail on it,
-then re-requesting containers on it will amplify the damage.
-
-## Actions
-
-### Bootstrap
-
-1. Persistent Role History file not found; empty data structures created.
-
-### Thaw
-
-When thawing, the Role History should be loaded -if it is missing Slider
-must revert to the bootstrap actions.
-
-If found, the Role History will contain Slider's view of the Slider Cluster's
-state at the time the history was saved, explicitly recording the last-used
-time of all nodes no longer hosting a role's container. By noting which roles
-were actually being served, it implicitly notes which nodes have a `last_used`
-value greater than any of the `last_used` fields persisted in the file. That is:
-all node entries listed as having active nodes at the time the history was
-saved must have more recent data than those nodes listed as inactive.
-
-When rebuilding the data structures, the fact that nodes were active at
-save time must be converted into the data that indicates that the nodes
-were at least in use *at the time the data was saved*. The state of the cluster
-after the last save is unknown.
-
-1: Role History loaded; Failure => Bootstrap.
-2: Future: if role list enum != current enum, remapping could take place. Until then: fail.
-3: Mark all nodes as active at save time to that of the
-
- //define a threshold
- threshold = rolehistory.saveTime - 7*24*60*60* 1000
-
-
- for (clusterId, clusternode) in rolehistory.clusterNodes().entries() :
- for (role, nodeEntry) in clusterNode.getNodeEntries():
- nodeEntry.requested = 0
- nodeEntry.releasing = 0
- if nodeEntry.active > 0 :
- nodeEntry.last_used = rolehistory.saveTime;
- nodeEntry.n.active = 0
- if nodeEntry.last_used < threshold :
- clusterNode.remove(role)
- else:
- availableNodes[role].add(clusterId)
- if clusterNode.getNodeEntries() isEmpty :
- rolehistory.clusterNodes.remove(clusterId)
-
-
- for availableNode in availableNodes:
- sort(availableNode,new last_used_comparator())
-
-After this operation, the structures are purged with all out of date entries,
-and the available node list contains a sorted list of the remainder.
-
-### AM Restart
-
-
-1: Create the initial data structures as the thaw operation
-2: update the structure with the list of live nodes, removing those nodes
-from the list of available nodes
-
- now = time()
- activeContainers = RM.getActiveContainers()
-
- for container in activeContainers:
- nodeId = container.nodeId
- clusterNode = roleHistory.nodemap.getOrCreate(nodeId)
- role = extractRoleId(container.getPriority)
- nodeEntry = clusterNode.getOrCreate(role)
- nodeEntry.active++
- nodeEntry.last_used = now
- availableNodes[role].remove(nodeId)
-
-There's no need to resort the available node list -all that has happened
-is that some entries have been removed
-
-
-**Issue**: what if requests come in for a `(role, requestID)` for
-the previous instance of the AM? Could we just always set the initial
-requestId counter to a random number and hope the collision rate is very, very
-low (2^24 * #(outstanding_requests)). If YARN-1041 ensures that
-a restarted AM does not receive outstanding requests, this issue goes away.
-
-
-### Teardown
-
-1. If dirty, save role history to its file.
-1. Issue release requests
-1. Maybe update data structures on responses, but do not mark Role History
-as dirty or flush it to disk.
-
-This strategy is designed to eliminate the expectation that there will ever
-be a clean shutdown -and so that the startup-time code should expect
-the Role History to have been written during shutdown. Instead the code
-should assume that the history was saved to disk at some point during the life
-of the Slider Cluster -ideally after the most recent change, and that the information
-in it is only an approximate about what the previous state of the cluster was.
-
-### Flex: Requesting a container in role `role`
-
-
- node = availableNodes[roleId].pop()
- if node != null :
- node.nodeEntry[roleId].requested++;
- outstanding = outstandingRequestTracker.addRequest(node, roleId)
- request.node = node
- request.priority = outstanding.priority
-
- //update existing Slider role status
- roleStatus[roleId].incRequested();
-
-
-There is a bias here towards previous nodes, even if the number of nodes
-in the cluster has changed. This is why a node is picked where the number
-of `active-releasing == 0 and requested == 0`, rather than where it is simply the lowest
-value of `active + requested - releasing`: if there is no node in the nodemap that
-is not running an instance of that role, it is left to the RM to decide where
-the role instance should be instantiated.
-
-This bias towards previously used nodes also means that (lax) requests
-will be made of nodes that are currently unavailable either because they
-are offline or simply overloaded with other work. In such circumstances,
-the node will have an active count of zero -so the search will find these
-nodes and request them -even though the requests cannot be satisfied.
-As a result, the request will be downgraded to a rack-local or cluster-wide,
-request -an acceptable degradation on a cluster where all the other entries
-in the nodemap have instances of that specific node -but not when there are
-empty nodes.
-
-
-#### Solutions
-
-1. Add some randomness in the search of the datastructure, rather than simply
-iterate through the values. This would prevent the same unsatisfiable
-node from being requested first.
-
-1. Keep track of requests, perhaps through a last-requested counter -and use
-this in the selection process. This would radically complicate the selection
-algorithm, and would not even distinguish "node recently released that was
-also the last requested" from "node that has not recently satisfied requests
-even though it was recently requested".
-
-1. Keep track of requests that weren't satisfied, so identify a node that
-isn't currently satisfying requests.
-
-
-#### History Issues
-
-Without using that history, there is a risk that a very old assignment
-is used in place of a recent one and the value of locality decreased.
-
-But there are consequences:
-
-**Performance**:
-
-Using the history to pick a recent node may increase selection times on a
-large cluster, as for every instance needed, a scan of all nodes in the
-nodemap is required (unless there is some clever bulk assignment list being built
-up), or a sorted version of the nodemap is maintained, with a node placed
-at the front of this list whenever its is updated.
-
-**Thaw-time problems**
-
-There is also the risk that while thawing, the `rolehistory.saved`
-flag may be updated while the cluster flex is in progress, so making the saved
-nodes appear out of date. Perhaps the list of recently released nodes could
-be rebuilt at thaw time.
-
-The proposed `recentlyReleasedList` addresses this, though it creates
-another data structure to maintain and rebuild at cluster thaw time
-from the last-used fields in the node entries.
-
-### AM Callback : onContainersAllocated
-
- void onContainersAllocated(List<Container> allocatedContainers)
-
-This is the callback received when containers have been allocated.
-Due to (apparently) race conditions, the AM may receive duplicate
-container allocations -Slider already has to recognize this and
-currently simply discards any surplus.
-
-If the AM tracks outstanding requests made for specific hosts, it
-will need to correlate allocations with the original requests, so as to decrement
-the node-specific request count. Decrementing the request count
-on the allocated node will not work, as the allocation may not be
-to the node originally requested.
-
- assignments = []
- operations = []
- for container in allocatedContainers:
- cid = container.getId();
- roleId = container.priority & 0xff
- nodeId = container.nodeId
- outstanding = outstandingRequestTracker.remove(C.priority)
- roleStatus = lookupRoleStatus(container);
- roleStatus.decRequested();
- allocated = roleStatus.incActual();
- if outstanding == null || allocated > desired :
- operations.add(new ContainerReleaseOperation(cid))
- surplusNodes.add(cid);
- surplusContainers++
- roleStatus.decActual();
- else:
- assignments.add(new ContainerAssignment(container, role))
- node = nodemap.getOrCreate(nodeId)
- nodeentry = node.get(roleId)
- if nodeentry == null :
- nodeentry = new NodeEntry()
- node[roleId] = nodeentry
- nodeentry.active = 1
- else:
- if nodeentry.requested > 0 :
- nodeentry.requested--
- nodeentry.active++
- nodemap.dirty = true
-
- // work back from request ID to node where the
- // request was outstanding
- requestID = outstanding != null? outstanding.nodeId : null
- if requestID != null:
- reqNode = nodeMap.get(requestID)
- reqNodeEntry = reqNode.get(roleId)
- reqNodeEntry.requested--
- if reqNodeEntry.available() :
- availableNodeList.insert(reqNodeEntry)
-
-
-
-1. At end of this, there is a node in the nodemap, which has recorded that
-there is now an active node entry for that role. The outstanding request has
-been removed.
-
-1. If a callback comes in for which there is no outstanding request, it is rejected
-(logged, ignored, etc). This handles duplicate responses as well as any
-other sync problem.
-
-1. The node selected for the original request has its request for a role instance
-decremented, so that it may be viewed as available again. The node is also
-re-inserted into the AvailableNodes list -not at its head, but at its position
-in the total ordering of the list.
-
-### NMClientAsync Callback: onContainerStarted()
-
-
- onContainerStarted(ContainerId containerId)
-
-The AM uses this as a signal to remove the container from the list
-of starting containers, moving it into the map of live nodes; the counters
-in the associated `RoleInstance` are updated accordingly; the node entry
-adjusted to indicate it has one more live node and one less starting node.
-
-
-### NMClientAsync Callback: onContainerStartFailed()
-
-
-The AM uses this as a signal to remove the container from the list
-of starting containers -the count of starting containers for the relevant
-NodeEntry is decremented. If the node is now available for instances of this
-container, it is returned to the queue of available nodes.
-
-
-### Flex: Releasing a role instance from the cluster
-
-Simple strategy: find a node with at least one active container
-
- select a node N in nodemap where for NodeEntry[roleId]: active > releasing;
- nodeentry = node.get(roleId)
- nodeentry.active--;
-
-Advanced Strategy:
-
- Scan through the map looking for a node where active >1 && active > releasing.
- If none are found, fall back to the previous strategy
-
-This is guaranteed to release a container on any node with >1 container in use,
-if such a node exists. If not, the scan time has increased to #(nodes).
-
-Once a node has been identified
-
-1. a container on it is located (via the existing container map). This container
-must: be of the target role, and not already be queued for release.
-1. A release operation is queued trigger a request for the RM.
-1. The (existing) `containersBeingReleased` Map has the container inserted into it
-
-After the AM processes the request, it triggers a callback
-
-### AM callback onContainersCompleted:
-
- void onContainersCompleted(List<ContainerStatus> completedContainers)
-
-This callback returns a list of containers that have completed.
-
-These need to be split into successful completion of a release request
-and containers which have failed.
-
-This is currently done by tracking which containers have been queued
-for release, as well as which were rejected as surplus before even having
-any role allocated onto them.
-
-A container is considered to have failed if it was an active container which
-has completed although it wasn't on the list of containers to release
-
- shouldReview = false
- for container in completedContainers:
- containerId = container.containerId
- nodeId = container.nodeId
- node = nodemap.get(nodeId)
- if node == null :
- // unknown node
- continue
- roleId = node.roleId
- nodeentry = node.get(roleId)
- nodeentry.active--
- nodemap.dirty = true
- if getContainersBeingReleased().containsKey(containerId) :
- // handle container completion
- nodeentry.releasing --
-
- // update existing Slider role status
- roleStatus[roleId].decReleasing();
- containersBeingReleased.remove(containerId)
- else:
- //failure of a live node
- roleStatus[roleId].decActual();
- shouldReview = true
-
- if nodeentry.available():
- nodentry.last_used = now()
- availableNodes[roleId].insert(node)
- //trigger a comparison of actual vs desired
- if shouldReview :
- reviewRequestAndReleaseNodes()
-
-By calling `reviewRequestAndReleaseNodes()` the AM triggers
-a re-evaluation of how many instances of each node a cluster has, and how many
-it needs. If a container has failed and that freed up all role instances
-on that node, it will have been inserted at the front of the `availableNodes` list.
-As a result, it is highly likely that a new container will be requested on
-the same node. (The only way a node the list would be newer is
-be if other containers were completed in the same callback)
-
-
-
-### Implementation Notes ###
-
-Notes made while implementing the design.
-
-`OutstandingRequestTracker` should also track requests made with
-no target node; this makes seeing what is going on easier. `ARMClientImpl`
-is doing something similar, on a priority-by-priority basis -if many
-requests are made, each with their own priority, that base class's hash tables
-may get overloaded. (it assumes a limited set of priorities)
-
-Access to the role history datastructures was restricted to avoid
-synchronization problems. Protected access is permitted so that a
-test subclass can examine (and change?) the internals.
-
-`NodeEntries need to add a launching value separate from active so that
-when looking for nodes to release, no attempt is made to release
-a node that has been allocated but is not yet live.
-
-We can't reliably map from a request to a response. Does that matter?
-If we issue a request for a host and it comes in on a different port, do we
-care? Yes -but only because we are trying to track nodes which have requests
-outstanding so as not to issue new ones. But if we just pop the entry
-off the available list, that becomes moot.
-
-Proposal: don't track the requesting numbers in the node entries, just
-in the role status fields.
-
-but: this means that we never re-insert nodes onto the available list if a
-node on them was requested but not satisfied.
-
-Other issues: should we place nodes on the available list as soon as all the entries
-have been released? I.e. Before YARN has replied
-
-RoleStats were removed -left in app state. Although the rolestats would
-belong here, leaving them where they were reduced the amount of change
-in the `AppState` class, so risk of something breaking.
-
-## MiniYARNCluster node IDs
-
-Mini YARN cluster NodeIDs all share the same hostname , at least when running
-against file://; so mini tests with >1 NM don't have a 1:1 mapping of
-`NodeId:NodeInstance`. What will happen is that
-`NodeInstance getOrCreateNodeInstance(Container container) '
-will always return the same (now shared) `NodeInstance`.
-
-## Releasing Containers when shrinking a cluster
-
-When identifying instances to release in a bulk downscale operation, the full
-list of targets must be identified together. This is not just to eliminate
-multiple scans of the data structures, but because the containers are not
-released until the queued list of actions are executed -the nodes' release-in-progress
-counters will not be incremented until after all the targets have been identified.
-
-It also needs to handle the scenario where there are many role instances on a
-single server -it should prioritize those.
-
-
-The NodeMap/NodeInstance/NodeEntry structure is adequate for identifying nodes,
-at least provided there is a 1:1 mapping of hostname to NodeInstance. But it
-is not enough to track containers in need of release: the AppState needs
-to be able to work backwards from a NodeEntry to container(s) stored there.
-
-The `AppState` class currently stores this data in a `ConcurrentMap<ContainerId, RoleInstance>`
-
-To map from NodeEntry/NodeInstance to containers to delete, means that either
-a new datastructure is created to identify containers in a role on a specific host
-(e.g a list of ContainerIds under each NodeEntry), or we add an index reference
-in a RoleInstance that identifies the node. We already effectively have that
-in the container
-
-### dropping any available nodes that are busy
-
-When scanning the available list, any nodes that are no longer idle for that
-role should be dropped from the list.
-
-This can happen when an instance was allocated on a different node from
-that requested.
-
-### Finding a node when a role has instances in the cluster but nothing
-known to be available
-
-One condition found during testing is the following:
-
-1. A role has one or more instances running in the cluster
-1. A role has no entries in its available list: there is no history of the
-role ever being on nodes other than which is currently in use.
-1. A new instance is requested.
-
-In this situation, the `findNodeForNewInstance` method returns null: there
-is no recommended location for placement. However, this is untrue: all
-nodes in the cluster `other` than those in use are the recommended nodes.
-
-It would be possible to build up a list of all known nodes in the cluster that
-are not running this role and use that in the request, effectively telling the
-AM to pick one of the idle nodes. By not doing so, we increase the probability
-that another instance of the same role will be allocated on a node in use,
-a probability which (were there capacity on these nodes and placement random), be
-`1/(clustersize-roleinstances)`. The smaller the cluster and the bigger the
-application, the higher the risk.
-
-This could be revisited, if YARN does not support anti-affinity between new
-requests at a given priority and existing ones: the solution would be to
-issue a relaxed placement request listing all nodes that are in the NodeMap and
-which are not running an instance of the specific role. [To be even more rigorous,
-the request would have to omit those nodes for which an allocation has already been
-made off the available list and yet for which no container has yet been
-granted].
-
-
-## Reworked Outstanding Request Tracker
-
-The reworked request tracker behaves as follows
-
-1. outstanding requests with specific placements are tracked by `(role, hostname)`
-1. container assigments are attempted to be resolved against the same parameters.
-1. If found: that request is considered satisfied *irrespective of whether or not
-the request that satisfied the allocation was the one that requested that location.
-1. When all instances of a specific role have been allocated, the hostnames of
-all outstanding requests are returned to the available node list on the basis
-that they have been satisifed elswhere in the YARN cluster. This list is
-then sorted.
-
-This strategy returns unused hosts to the list of possible hosts, while retaining
-the ordering of that list in most-recent-first.
-
-### Weaknesses
-
-if one or more container requests cannot be satisifed, then all the hosts in
-the set of outstanding requests will be retained, so all these hosts in the
-will be considered unavailable for new location-specific requests.
-This may imply that new requests that could be explicity placed will now only
-be randomly placed -however, it is moot on the basis that if there are outstanding
-container requests it means the RM cannot grant resources: new requests at the
-same priority (i.e. same Slider Role ID) will not be granted either.
-
-The only scenario where this would be different is if the resource requirements
-of instances of the target role were decreated during a cluster flex such that
-the placement could now be satisfied on the target host. This is not considered
-a significant problem.
-
-# Persistence
-
-The initial implementation uses the JSON-formatted Avro format; while significantly
-less efficient than a binary format, it is human-readable
-
-Here are sequence of entries from a test run on a single node cluster; running 1 HBase Master
-and two region servers.
-
-Initial save; the instance of Role 1 (HBase master) is live, Role 2 (RS) is not.
-
- {"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183475949,"savedx":"14247c3aeed","roles":3}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":0}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":0}}}
-
-At least one RS is live:
-
- {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183476010,"savedx":"14247c3af2a","roles":3}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":0}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":0}}}
-
-Another entry is saved -presumably the second RS is now live, which triggered another write
-
- {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183476028,"savedx":"14247c3af3c","roles":3}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":0}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":0}}}
-
-At this point the cluster was frozen and thawed. Slider does not save the cluster state
-at freeze time, but does as it is rebuilt.
-
-When the cluster is restarted, every node that was active for a role at the time the file was saved `1384183476028`
-is given a last_used timestamp of that time.
-
-When the history is next saved, the master has come back onto the (single) node,
-it is active while its `last_used` timestamp is the previous file's timestamp.
-No region servers are yet live.
-
- {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183512173,"savedx":"14247c43c6d","roles":3}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":1384183476028}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":false,"last_used":1384183476028}}}
-
-Here a region server is live
-
- {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183512199,"savedx":"14247c43c87","roles":3}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":1384183476028}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":1384183476028}}}
-
-And here, another region server has started. This does not actually change the contents of the file
-
- {"entry":{"org.apache.hoya.avro.RoleHistoryFooter":{"count":2}}}{"entry":{"org.apache.hoya.avro.RoleHistoryHeader":{"version":1,"saved":1384183512217,"savedx":"14247c43c99","roles":3}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":1,"active":true,"last_used":1384183476028}}}
- {"entry":{"org.apache.hoya.avro.NodeEntryRecord":{"host":"192.168.1.85","role":2,"active":true,"last_used":1384183476028}}}
-
-The `last_used` timestamps will not be changed until the cluster is shrunk or thawed, as the `active` flag being set
-implies that the server is running both roles at the save time of `1384183512217`.
-
-## Resolved issues
-
-> How best to distinguish at thaw time from nodes used just before thawing
-from nodes used some period before? Should the RoleHistory simply forget
-about nodes which are older than some threshold when reading in the history?
-
-we just track last used times
-
-
-> Is there a way to avoid tracking the outstanding requests?
-
-No
-
-> What will the strategy of picking the most-recently-used node do if
-that node creates the container and then fails to start it up. Do we need
-to add blacklisting too? Or actually monitor the container start time, and
-if a container hasn't been there for very long, don't pick it.
-
-Startup failures drop the node from the ready-to-use list; the node is no longer
-trusted. We don't blacklist it (yet)
-
-
-> Should we prioritise a node that was used for a long session ahead of
-a node that was used more recently for a shorter session? Maybe, but
-it complicates selection as generating a strict order of nodes gets
-significantly harder.
-
-No: you need to start tracking aggregate execution time, for the last session.
-In a stable state, all servers recorded in the history will have spread the
-data amongst them, so its irrelevant.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/client-configuration.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/client-configuration.md b/src/site/markdown/client-configuration.md
deleted file mode 100644
index f3a99ce..0000000
--- a/src/site/markdown/client-configuration.md
+++ /dev/null
@@ -1,310 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Client Configuration
-
-This document covers how the client application is itself configured.
-
-## Summary
-
-The client application can be configured
-
-1. On the command line, which can set client options and JVM system properties.
-2. With Hadoop-style configuration options in the file `slider-client.xml`
- in the configuration directory`conf/` dir
-2. Or, if the environment variable `SLIDER_CONF_DIR` is set, in the
- file `$SLIDER_CONF_DIR/slider-client.xml`
-1. Logging is defined in the `log4j.properties` file in the same configuration
-directory.
-1. VM options can be defined in `SLIDER_JVM_OPTS`
-
-The options defined in a Slider cluster configuration are only used by the client
-when creating a cluster -not for the actual client itself.
-
-## Introduction
-
-The Slider client needs to be configured to talk to a Hadoop filesystem and a
-YARN resource manager ("the RM"). In a secure cluster it needs to be told the Kerberos
-identity, the *principal* of both the HDFS namenode and the YARN RM -and it may
-also need some JVM options set in order for Java's Kerberos module to
-correctly identify itself to these services.
-
-It cannot rely on local `HADOOP_PREFIX/conf/hadoop-site.xml` and
-`$YARN_PREFIX/conf/yarn-site.xml` files -because it is designed to
-work on client machines that may not have Hadoop and YARN installed.
-
-Instead all client-side (non-JVM) options can be predefined in the
-configuration file `slider-client.xml`.
-
-## Setting Slider JVM options
-
-Core JVM options can be set in the environment variable `SLIDER_JVM_OPTS`;
-if unset the `bin/slider` script will use the default values that were
-current when that version of Slider was released. These values may change
-across versions, and may in fact be.
-
-At the time of writing, the default values were:
-
- "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=${confdir}"
-
-To allow some Java system properties to be set without editing this
-environment variable, such system properties may be set on the Slider command
-line through the `-S` parameter. For example, the following two operations are
-equivalent in terms of setting the system property `java.security.krb5.realm`
-to the value `LOCAL`.
-
- export SLIDER_JVM_OPTS="-Djava.security.krb5.realm=LOCAL"
-
-and
-
- slider -S java.security.krb5.realm=LOCAL
-
-Note that the first declaration invalidates all default JVM options; if any of
-those were desired, they should be included in the new definition.
-
-Multiple system property declarations are allowed on the command line -including
-duplicate declarations. In such a case the order of assignment is undefined.
-
-For any system property that the user expects to have to issue on every command
--including any kerberos-related properties, adding them to the JVM options
-environment variable guarantees that they are always set.
-
-## Setting Slider client options on the command line with the `-D` parameter
-
-The slider client is configured via Hadoop-style configuration options.
-To be precise, all standard Hadoop-common, hadoop-hdfs client and hadoop-yar
-client-side options control how Slider communicates with the Hadoop YARN cluster.
-
-There are extra options specific to Slider itself, options which
-are again set as Hadoop configuration parameters.
-
-All Hadoop and Slider options can be set on the command line using the `-D`
-parameter followed by the appropriate `key=value` argument
-
-
-For example, here is a definition of the default Hadoop filesystem:
-
- -D fs.defaultFS=hdfs://namenode:9000
-
-Multiple definitions are of course allowed on the command line
-
- -D fs.defaultFS=hdfs://namenode:9000 -D dfs.namenode.kerberos.principal=hdfs/namenode@LOCAL
-
-Slider-specific options can be made the same way
-
- -D slider.kerberos.principal=
-
-If duplicate declarations are made the order of assignment is undefined.
-
-# Setting common options through specific command-line arguments
-
-Some Hadoop and Slider options are so common that they have specific
-shortcut commands to aid their use
-
-`-m`, `--manager` : sets the YARN resource manager. Equivalent to setting the
-`yarn.resourcemanager.address` option
-
-`--fs`, `--filesystem`: defines the filesystem. Equivalent to setting the
-`fs.defaultFS` option
-
-If these shortcuts are used and the options are also defined via `-D`
-declarations, the order of assignment is undefined.
-
-# Defining Hadoop and Slider Options in the `slider-client.xml` file.
-
-In the Slider installation, alongside the `bin/slider` script is
-a configuration directory `conf`. This contains the files:
-
-1. `log4j.properties`
-1. `slider-client.xml`
-
-The `log4j.properties` file is not covered here -it is a standard Log4J file.
-At the time of writing, this log configuration file is used on both the
-client and the server.
-
-The `slider-client.xml` file is a hadoop-formatted XML options file, which
-is read by the Slider client -but not by they Slider Application Master.
-
-Here is an example file:
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>namenode:8033</value>
- </property>
-
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://namenode:9000</value>
- </property>
-
- <property>
- <name>ipc.client.fallback-to-simple-auth-allowed</name>
- <value>false</value>
- </property>
-
-
-This defines both the filesystem and the YARN RM, and so obviates the need
-to declare either on the command line.
-
-If an option is defined in the `slider-client.xml` file and on the command line
--be it by a `-D key=value` declaration or a `--manager` or `--filesystem`
-definition. (this holds even if the value is declared with `<final>true</final>`).
-
-## Selecting an alternate Slider configuration directory
-
-The environment variable `SLIDER_CONF_DIR` can be used to declare an alternate
-configuration directory. If set, the directory it identifies will be used
-as the source of the `log4j.properties` and `slider-client.xml` files.
-
-## Slider Client Configuration options
-
-As well as standard YARN and Hadoop configuration options, Slider supports
-a limited number of slider-specific configuration parameters.
-
- <property>
- <name>slider.zookeeper.quorum</name>
- <value>localhost:2181,zookeeper2:4545</value>
- </property>
-
- <property>
- <name>slider.yarn.queue</name>
- <value>default</value>
- </property>
-
- <property>
- <name>slider.security.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <name>slider.yarn.queue</name>
- <value>default</value>
- </property>
-
- <property>
- <name>slider.yarn.queue.priority</name>
- <value>1</value>
- </property>
-
- <property>
- <name>slider.yarn.restart.limit</name>
- <value>5</value>
- <description>How many times to start/restart the Slider AM</description>
- </property>
-
- <property>
- <name>slider.cluster.directory.permissions</name>
- <value>750</value>
- </property>
-
- <property>
- <name>slider.data.directory.permissions</name>
- <value>750</value>
- </property>
-
-### `slider.zookeeper.quorum` - the zookeeper quorum.
-
-This defines the zookeeper quorum for this YARN cluster.
-
-It is used to locate the service registry, enable running instances to publish
-information about their application, and for clients to query this.
-
-It is also used as the default zookeeper binding for any application that
-uses zookeeper in its configuration -the value set when the application is
-defined will be copied into the instance definition file.
-
-### `"slider.registry.path"` - the zookeeper path for the service registry
-
-This declares the the zookeeper path for the service registry.
-
-### `slider.security.enabled` - enable security.
-
-This turns security on; consult [Security](security.html) for more information.
-
-
-### `slider.yarn.restart.limit` - set limit on Application Master Restarts
-
-This limits how many times YARN should start a failed application master.
-
-A short restart limit is useful when initially creating a cluster, as it
-ensures that YARN does not repeatedly try to restart a failing application.
-
-In production, however, a large number prevents YARN from halting a Slider
-application merely because failures in the underlying YARN cluster have
-triggered restarts.
-
-*Important:* The cluster-wide limit of `yarn.resourcemanager.am.max-attempts`
-places an upper limit on the number of retries that any application can request.
-If the application fails after less restarts than requested, check this cluster
-setting.
-
-### `slider.yarn.queue` - the name of the YARN queue for the cluster.
-
-This identifies the queue submit the application creation request to, which can
-define the priority, resource limits and other values of an application. All
-containers created in the Slider cluster will share this same queue.
-
-Default value: `default`.
-
-### `slider.yarn.queue.priority` - the name of the YARN queue for the cluster.
-
-This identifies the priority within the queue. The lower the value, the higher the
-priority
-
-Default value: `1`.
-
- bin/slider thaw cl1 -D slider.yarn.queue.priority=5
-
-
-
-#### `slider.cluster.directory.permissions`
-
-An octal-format (`chmod`-style) permissions mask for the directory
-that contains the cluster specification `${user.home}/.slider/clusters/${clustername}`
-
- <property>
- <name>slider.cluster.directory.permissions</name>
- <value>750</value>
- </property>
-
-#### `slider.data.directory.permissions`
-
-An octal-format (`chmod`-style) permissions mask for the directory
-that contains the application data `${user.home}/.slider/clusters/${clustername}/database`
-
- <property>
- <name>slider.data.directory.permissions</name>
- <value>750</value>
- </property>
-
-
-## Debugging configuration issues
-
-If the slider packages are set to log at debug level in the log4j configuration
-file, details on properties will be part of the copious output.
-
-
-## How client options are passed down to created clusters.
-
-Apart from the filesystem bindings, Client configuration options are
-not passed down to the XML site specification of the created cluster.
-
-The sole options passed down are the HDFS bindings: `fs.defaultFS`,
-which is passed down both as that property and as `fs.default.name`,
-and, in a secure cluster, the security flag (`slider.security.enabled`)
-and the HDFS Kerberos principal.
-
[14/50] [abbrv] git commit: SLIDER-106 funtest properties skipped by
default
Posted by st...@apache.org.
SLIDER-106 funtest properties skipped by default
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/153f5e94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/153f5e94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/153f5e94
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 153f5e94b64a45e97c4b0494755472788455d678
Parents: 537d87c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 16:19:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 16:19:49 2014 -0700
----------------------------------------------------------------------
slider-funtest/pom.xml | 34 ++++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/153f5e94/slider-funtest/pom.xml
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 42a57b2..51b5cec 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -265,5 +265,39 @@
</dependencies>
+ <profiles>
+ <profile>
+ <id>tests-on-from-CLI</id>
+ <activation>
+ <property>
+ <name>slider.conf.dir</name>
+ </property>
+ </activation>
+ <properties>
+ <maven.test.skip>false</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-on-from-build.properties</id>
+ <activation>
+ <file>
+ <exists>../build.properties</exists>
+ </file>
+ </activation>
+ <properties>
+ <maven.test.skip>false</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-off</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <maven.test.skip>true</maven.test.skip>
+ </properties>
+ </profile>
+
+ </profiles>
</project>
[07/50] [abbrv] git commit: SLIDER-157 Remove app-packages/hbase-v0_96
Posted by st...@apache.org.
SLIDER-157 Remove app-packages/hbase-v0_96
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7729653f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7729653f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7729653f
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 7729653f73ab4208945794bde7e9f7f22fe89aa3
Parents: 57a9c9f
Author: tedyu <yu...@gmail.com>
Authored: Mon Jun 23 09:07:16 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Jun 23 09:07:16 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-v0_96/README.txt | 33 --
app-packages/hbase-v0_96/appConfig.json | 67 ----
.../hbase-v0_96/configuration/global.xml | 160 --------
.../hbase-v0_96/configuration/hbase-log4j.xml | 142 --------
.../hbase-v0_96/configuration/hbase-policy.xml | 53 ---
.../hbase-v0_96/configuration/hbase-site.xml | 365 -------------------
app-packages/hbase-v0_96/ganglia_metrics.json | 38 --
app-packages/hbase-v0_96/jmx_metrics.json | 56 ---
app-packages/hbase-v0_96/metainfo.xml | 109 ------
.../hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE | 16 -
.../hbase-v0_96/package/scripts/__init__.py | 19 -
.../hbase-v0_96/package/scripts/functions.py | 40 --
.../hbase-v0_96/package/scripts/hbase.py | 125 -------
.../hbase-v0_96/package/scripts/hbase_client.py | 43 ---
.../hbase-v0_96/package/scripts/hbase_master.py | 63 ----
.../package/scripts/hbase_regionserver.py | 66 ----
.../package/scripts/hbase_service.py | 45 ---
.../hbase-v0_96/package/scripts/params.py | 109 ------
.../package/scripts/status_params.py | 26 --
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 62 ----
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 62 ----
.../package/templates/hbase-env.sh.j2 | 81 ----
.../package/templates/hbase_client_jaas.conf.j2 | 22 --
.../package/templates/hbase_master_jaas.conf.j2 | 25 --
.../templates/hbase_regionserver_jaas.conf.j2 | 25 --
.../package/templates/regionservers.j2 | 20 -
app-packages/hbase-v0_96/resources.json | 19 -
app-packages/hbase/README.txt | 8 +-
28 files changed, 7 insertions(+), 1892 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/README.txt b/app-packages/hbase-v0_96/README.txt
deleted file mode 100644
index 33c93df..0000000
--- a/app-packages/hbase-v0_96/README.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-How to create a Slider package?
-
-Replace the placeholder tarball for HBase.
- cp ~/Downloads/hbase-0.96.1-hadoop2-bin.tar.gz package/files/
- rm package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
-
-Create a zip package at the root of the package (<slider enlistment>/app-packages/hbase-v0_96/)
- zip -r hbase_v096.zip .
-
-Verify the content using
- unzip -l "$@" hbase_v096.zip
-
-While appConfig.json and resources.json are not required for the package they work
-well as the default configuration for Slider apps. So its advisable that when you
-create an application package for Slider, include sample/default resources.json and
-appConfig.json for a minimal Yarn cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/appConfig.json b/app-packages/hbase-v0_96/appConfig.json
deleted file mode 100644
index 1d828e2..0000000
--- a/app-packages/hbase-v0_96/appConfig.json
+++ /dev/null
@@ -1,67 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "agent.conf": "/slider/agent/conf/agent.ini",
- "application.def": "/slider/hbase_v096.zip",
- "config_types": "core-site,hdfs-site,hbase-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/hbase-0.96.1-hadoop2-bin.tar.gz",
- "site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-0.96.1-hadoop2",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.global.hbase_master_heapsize": "1024m",
- "site.global.hbase_regionserver_heapsize": "1024m",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8667",
- "site.global.ganglia_server_id": "Application1",
- "site.hbase-site.hbase.hstore.flush.retries.number": "120",
- "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
- "site.hbase-site.hbase.hstore.compactionThreshold": "3",
- "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
- "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
- "site.hbase-site.hbase.regionserver.handler.count": "60",
- "site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "site.hbase-site.hbase.hregion.memstore.block.multiplier": "2",
- "site.hbase-site.hbase.hregion.memstore.flush.size": "134217728",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.zookeeper.property.clientPort": "2181",
- "site.hbase-site.hbase.regionserver.global.memstore.upperLimit": "0.4",
- "site.hbase-site.zookeeper.session.timeout": "30000",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
- "site.hbase-site.hbase.hregion.max.filesize": "10737418240",
- "site.hbase-site.hfile.block.cache.size": "0.40",
- "site.hbase-site.hbase.security.authentication": "simple",
- "site.hbase-site.hbase.defaults.for.version.skip": "true",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "/hbase-unsecure",
- "site.hbase-site.hbase.hstore.blockingStoreFiles": "10",
- "site.hbase-site.hbase.hregion.majorcompaction": "86400000",
- "site.hbase-site.hbase.security.authorization": "false",
- "site.hbase-site.hbase.cluster.distributed": "true",
- "site.hbase-site.hbase.hregion.memstore.mslab.enabled": "true",
- "site.hbase-site.hbase.client.scanner.caching": "100",
- "site.hbase-site.hbase.zookeeper.useMulti": "true",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0",
- "site.core-site.fs.defaultFS": "${NN_URI}",
- "site.hdfs-site.dfs.namenode.https-address": "${NN_HOST}:50470",
- "site.hdfs-site.dfs.namenode.http-address": "${NN_HOST}:50070"
- },
- "components": {
- "HBASE_MASTER": {
- },
- "slider-appmaster": {
- "jvm.heapsize": "256M"
- },
- "HBASE_REGIONSERVER": {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/configuration/global.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/configuration/global.xml b/app-packages/hbase-v0_96/configuration/global.xml
deleted file mode 100644
index b2c57bd..0000000
--- a/app-packages/hbase-v0_96/configuration/global.xml
+++ /dev/null
@@ -1,160 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>hbasemaster_host</name>
- <value></value>
- <description>HBase Master Host.</description>
- </property>
- <property>
- <name>regionserver_hosts</name>
- <value></value>
- <description>Region Server Hosts</description>
- </property>
- <property>
- <name>hbase_log_dir</name>
- <value>/var/log/hbase</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_pid_dir</name>
- <value>/var/run/hbase</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_log_dir</name>
- <value>/var/log/hbase</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_regionserver_heapsize</name>
- <value>1024</value>
- <description>Log Directories for HBase.</description>
- </property>
- <property>
- <name>hbase_master_heapsize</name>
- <value>1024</value>
- <description>HBase Master Heap Size</description>
- </property>
- <property>
- <name>hstore_compactionthreshold</name>
- <value>3</value>
- <description>HBase HStore compaction threshold.</description>
- </property>
- <property>
- <name>hfile_blockcache_size</name>
- <value>0.40</value>
- <description>HFile block cache size.</description>
- </property>
- <property>
- <name>hstorefile_maxsize</name>
- <value>10737418240</value>
- <description>Maximum HStoreFile Size</description>
- </property>
- <property>
- <name>regionserver_handlers</name>
- <value>60</value>
- <description>HBase RegionServer Handler</description>
- </property>
- <property>
- <name>hregion_majorcompaction</name>
- <value>604800000</value>
- <description>The time between major compactions of all HStoreFiles in a region. Set to 0 to disable automated major compactions.</description>
- </property>
- <property>
- <name>hregion_blockmultiplier</name>
- <value>2</value>
- <description>HBase Region Block Multiplier</description>
- </property>
- <property>
- <name>hregion_memstoreflushsize</name>
- <value></value>
- <description>HBase Region MemStore Flush Size.</description>
- </property>
- <property>
- <name>client_scannercaching</name>
- <value>100</value>
- <description>Base Client Scanner Caching</description>
- </property>
- <property>
- <name>zookeeper_sessiontimeout</name>
- <value>30000</value>
- <description>ZooKeeper Session Timeout</description>
- </property>
- <property>
- <name>hfile_max_keyvalue_size</name>
- <value>10485760</value>
- <description>HBase Client Maximum key-value Size</description>
- </property>
- <property>
- <name>hbase_hdfs_root_dir</name>
- <value>/apps/hbase/data</value>
- <description>HBase Relative Path to HDFS.</description>
- </property>
- <property>
- <name>hbase_conf_dir</name>
- <value>/etc/hbase</value>
- <description>Config Directory for HBase.</description>
- </property>
- <property>
- <name>hdfs_enable_shortcircuit_read</name>
- <value>true</value>
- <description>HDFS Short Circuit Read</description>
- </property>
- <property>
- <name>hdfs_support_append</name>
- <value>true</value>
- <description>HDFS append support</description>
- </property>
- <property>
- <name>hstore_blockingstorefiles</name>
- <value>10</value>
- <description>HStore blocking storefiles.</description>
- </property>
- <property>
- <name>regionserver_memstore_lab</name>
- <value>true</value>
- <description>Region Server memstore.</description>
- </property>
- <property>
- <name>regionserver_memstore_lowerlimit</name>
- <value>0.38</value>
- <description>Region Server memstore lower limit.</description>
- </property>
- <property>
- <name>regionserver_memstore_upperlimit</name>
- <value>0.4</value>
- <description>Region Server memstore upper limit.</description>
- </property>
- <property>
- <name>hbase_conf_dir</name>
- <value>/etc/hbase</value>
- <description>HBase conf dir.</description>
- </property>
- <property>
- <name>hbase_user</name>
- <value>hbase</value>
- <description>HBase User Name.</description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/configuration/hbase-log4j.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/configuration/hbase-log4j.xml b/app-packages/hbase-v0_96/configuration/hbase-log4j.xml
deleted file mode 100644
index 3bbc549..0000000
--- a/app-packages/hbase-v0_96/configuration/hbase-log4j.xml
+++ /dev/null
@@ -1,142 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
-
- <property>
- <name>content</name>
- <value>
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Define some default values that can be overridden by system properties
-hbase.root.logger=INFO,console
-hbase.security.logger=INFO,console
-hbase.log.dir=.
-hbase.log.file=hbase.log
-
-# Define the root logger to the system property "hbase.root.logger".
-log4j.rootLogger=${hbase.root.logger}
-
-# Logging Threshold
-log4j.threshold=ALL
-
-#
-# Daily Rolling File Appender
-#
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-#log4j.appender.DRFA.MaxBackupIndex=30
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-# Rolling File Appender properties
-hbase.log.maxfilesize=256MB
-hbase.log.maxbackupindex=20
-
-# Rolling File Appender
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}
-
-log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}
-log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-#
-# Security audit appender
-#
-hbase.security.log.file=SecurityAuth.audit
-hbase.security.log.maxfilesize=256MB
-hbase.security.log.maxbackupindex=20
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}
-log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.category.SecurityLogger=${hbase.security.logger}
-log4j.additivity.SecurityLogger=false
-#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
-
-#
-# Null Appender
-#
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
-
-# Custom Logging levels
-
-log4j.logger.org.apache.zookeeper=INFO
-#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
-log4j.logger.org.apache.hadoop.hbase=DEBUG
-# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
-log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
-log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
-#log4j.logger.org.apache.hadoop.dfs=DEBUG
-# Set this class to log INFO only otherwise its OTT
-# Enable this to get detailed connection error/retry logging.
-# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
-
-
-# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
-#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
-
-# Uncomment the below if you want to remove logging of client region caching'
-# and scan of .META. messages
-# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
-# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
-
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/configuration/hbase-policy.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/configuration/hbase-policy.xml b/app-packages/hbase-v0_96/configuration/hbase-policy.xml
deleted file mode 100644
index e45f23c..0000000
--- a/app-packages/hbase-v0_96/configuration/hbase-policy.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
- <description>ACL for HRegionInterface protocol implementations (ie.
- clients talking to HRegionServers)
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.admin.protocol.acl</name>
- <value>*</value>
- <description>ACL for HMasterInterface protocol implementation (ie.
- clients talking to HMaster for admin operations).
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-
- <property>
- <name>security.masterregion.protocol.acl</name>
- <value>*</value>
- <description>ACL for HMasterRegionInterface protocol implementations
- (for HRegionServers communicating with HMaster)
- The ACL is a comma-separated list of user and group names. The user and
- group list is separated by a blank. For e.g. "alice,bob users,wheel".
- A special value of "*" means all users are allowed.</description>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/configuration/hbase-site.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/configuration/hbase-site.xml b/app-packages/hbase-v0_96/configuration/hbase-site.xml
deleted file mode 100644
index cf9416e..0000000
--- a/app-packages/hbase-v0_96/configuration/hbase-site.xml
+++ /dev/null
@@ -1,365 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <property>
- <name>hbase.rootdir</name>
- <value>hdfs://localhost:8020/apps/hbase/data</value>
- <description>The directory shared by region servers and into
- which HBase persists. The URL should be 'fully-qualified'
- to include the filesystem scheme. For example, to specify the
- HDFS directory '/hbase' where the HDFS instance's namenode is
- running at namenode.example.org on port 9000, set this value to:
- hdfs://namenode.example.org:9000/hbase. By default HBase writes
- into /tmp. Change this configuration else all data will be lost
- on machine restart.
- </description>
- </property>
- <property>
- <name>hbase.cluster.distributed</name>
- <value>true</value>
- <description>The mode the cluster will be in. Possible values are
- false for standalone mode and true for distributed mode. If
- false, startup will run all HBase and ZooKeeper daemons together
- in the one JVM.
- </description>
- </property>
- <property>
- <name>hbase.tmp.dir</name>
- <value>/hadoop/hbase</value>
- <description>Temporary directory on the local filesystem.
- Change this setting to point to a location more permanent
- than '/tmp' (The '/tmp' directory is often cleared on
- machine restart).
- </description>
- </property>
- <property>
- <name>hbase.local.dir</name>
- <value>${hbase.tmp.dir}/local</value>
- <description>Directory on the local filesystem to be used as a local storage
- </description>
- </property>
- <property>
- <name>hbase.master.info.bindAddress</name>
- <value></value>
- <description>The bind address for the HBase Master web UI
- </description>
- </property>
- <property>
- <name>hbase.master.info.port</name>
- <value></value>
- <description>The port for the HBase Master web UI.</description>
- </property>
- <property>
- <name>hbase.regionserver.info.port</name>
- <value></value>
- <description>The port for the HBase RegionServer web UI.</description>
- </property>
- <property>
- <name>hbase.regionserver.global.memstore.upperLimit</name>
- <value>0.4</value>
- <description>Maximum size of all memstores in a region server before new
- updates are blocked and flushes are forced. Defaults to 40% of heap
- </description>
- </property>
- <property>
- <name>hbase.regionserver.handler.count</name>
- <value>60</value>
- <description>Count of RPC Listener instances spun up on RegionServers.
- Same property is used by the Master for count of master handlers.
- Default is 10.
- </description>
- </property>
- <property>
- <name>hbase.hregion.majorcompaction</name>
- <value>86400000</value>
- <description>The time (in milliseconds) between 'major' compactions of all
- HStoreFiles in a region. Default: 1 day.
- Set to 0 to disable automated major compactions.
- </description>
- </property>
-
- <property>
- <name>hbase.regionserver.global.memstore.lowerLimit</name>
- <value>0.38</value>
- <description>When memstores are being forced to flush to make room in
- memory, keep flushing until we hit this mark. Defaults to 35% of heap.
- This value equal to hbase.regionserver.global.memstore.upperLimit causes
- the minimum possible flushing to occur when updates are blocked due to
- memstore limiting.
- </description>
- </property>
- <property>
- <name>hbase.hregion.memstore.block.multiplier</name>
- <value>2</value>
- <description>Block updates if memstore has hbase.hregion.memstore.block.multiplier
- time hbase.hregion.flush.size bytes. Useful preventing
- runaway memstore during spikes in update traffic. Without an
- upper-bound, memstore fills such that when it flushes the
- resultant flush files take a long time to compact or split, or
- worse, we OOME
- </description>
- </property>
- <property>
- <name>hbase.hregion.memstore.flush.size</name>
- <value>134217728</value>
- <description>
- Memstore will be flushed to disk if size of the memstore
- exceeds this number of bytes. Value is checked by a thread that runs
- every hbase.server.thread.wakefrequency.
- </description>
- </property>
- <property>
- <name>hbase.hregion.memstore.mslab.enabled</name>
- <value>true</value>
- <description>
- Enables the MemStore-Local Allocation Buffer,
- a feature which works to prevent heap fragmentation under
- heavy write loads. This can reduce the frequency of stop-the-world
- GC pauses on large heaps.
- </description>
- </property>
- <property>
- <name>hbase.hregion.max.filesize</name>
- <value>10737418240</value>
- <description>
- Maximum HStoreFile size. If any one of a column families' HStoreFiles has
- grown to exceed this value, the hosting HRegion is split in two.
- Default: 1G.
- </description>
- </property>
- <property>
- <name>hbase.client.scanner.caching</name>
- <value>100</value>
- <description>Number of rows that will be fetched when calling next
- on a scanner if it is not served from (local, client) memory. Higher
- caching values will enable faster scanners but will eat up more memory
- and some calls of next may take longer and longer times when the cache is empty.
- Do not set this value such that the time between invocations is greater
- than the scanner timeout; i.e. hbase.regionserver.lease.period
- </description>
- </property>
- <property>
- <name>zookeeper.session.timeout</name>
- <value>30000</value>
- <description>ZooKeeper session timeout.
- HBase passes this to the zk quorum as suggested maximum time for a
- session (This setting becomes zookeeper's 'maxSessionTimeout'). See
- http://hadoop.apache.org/zookeeper/docs/current/zookeeperProgrammers.html#ch_zkSessions
- "The client sends a requested timeout, the server responds with the
- timeout that it can give the client. " In milliseconds.
- </description>
- </property>
- <property>
- <name>hbase.client.keyvalue.maxsize</name>
- <value>10485760</value>
- <description>Specifies the combined maximum allowed size of a KeyValue
- instance. This is to set an upper boundary for a single entry saved in a
- storage file. Since they cannot be split it helps avoiding that a region
- cannot be split any further because the data is too large. It seems wise
- to set this to a fraction of the maximum region size. Setting it to zero
- or less disables the check.
- </description>
- </property>
- <property>
- <name>hbase.hstore.compactionThreshold</name>
- <value>3</value>
- <description>
- If more than this number of HStoreFiles in any one HStore
- (one HStoreFile is written per flush of memstore) then a compaction
- is run to rewrite all HStoreFiles files as one. Larger numbers
- put off compaction but when it runs, it takes longer to complete.
- </description>
- </property>
- <property>
- <name>hbase.hstore.flush.retries.number</name>
- <value>120</value>
- <description>
- The number of times the region flush operation will be retried.
- </description>
- </property>
-
- <property>
- <name>hbase.hstore.blockingStoreFiles</name>
- <value>10</value>
- <description>
- If more than this number of StoreFiles in any one Store
- (one StoreFile is written per flush of MemStore) then updates are
- blocked for this HRegion until a compaction is completed, or
- until hbase.hstore.blockingWaitTime has been exceeded.
- </description>
- </property>
- <property>
- <name>hfile.block.cache.size</name>
- <value>0.40</value>
- <description>
- Percentage of maximum heap (-Xmx setting) to allocate to block cache
- used by HFile/StoreFile. Default of 0.25 means allocate 25%.
- Set to 0 to disable but it's not recommended.
- </description>
- </property>
-
- <!-- The following properties configure authentication information for
- HBase processes when using Kerberos security. There are no default
- values, included here for documentation purposes -->
- <property>
- <name>hbase.master.keytab.file</name>
- <value></value>
- <description>Full path to the kerberos keytab file to use for logging in
- the configured HMaster server principal.
- </description>
- </property>
- <property>
- <name>hbase.master.kerberos.principal</name>
- <value></value>
- <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
- that should be used to run the HMaster process. The principal name should
- be in the form: user/hostname@DOMAIN. If "_HOST" is used as the hostname
- portion, it will be replaced with the actual hostname of the running
- instance.
- </description>
- </property>
- <property>
- <name>hbase.regionserver.keytab.file</name>
- <value></value>
- <description>Full path to the kerberos keytab file to use for logging in
- the configured HRegionServer server principal.
- </description>
- </property>
- <property>
- <name>hbase.regionserver.kerberos.principal</name>
- <value></value>
- <description>Ex. "hbase/_HOST@EXAMPLE.COM". The kerberos principal name
- that should be used to run the HRegionServer process. The principal name
- should be in the form: user/hostname@DOMAIN. If "_HOST" is used as the
- hostname portion, it will be replaced with the actual hostname of the
- running instance. An entry for this principal must exist in the file
- specified in hbase.regionserver.keytab.file
- </description>
- </property>
-
- <!-- Additional configuration specific to HBase security -->
- <property>
- <name>hbase.superuser</name>
- <value>hbase</value>
- <description>List of users or groups (comma-separated), who are allowed
- full privileges, regardless of stored ACLs, across the cluster.
- Only used when HBase security is enabled.
- </description>
- </property>
-
- <property>
- <name>hbase.security.authentication</name>
- <value>simple</value>
- <description> Controls whether or not secure authentication is enabled for HBase. Possible values are 'simple'
- (no authentication), and 'kerberos'.
- </description>
- </property>
-
- <property>
- <name>hbase.security.authorization</name>
- <value>false</value>
- <description>Enables HBase authorization. Set the value of this property to false to disable HBase authorization.
- </description>
- </property>
-
- <property>
- <name>hbase.coprocessor.region.classes</name>
- <value></value>
- <description>A comma-separated list of Coprocessors that are loaded by
- default on all tables. For any override coprocessor method, these classes
- will be called in order. After implementing your own Coprocessor, just put
- it in HBase's classpath and add the fully qualified class name here.
- A coprocessor can also be loaded on demand by setting HTableDescriptor.
- </description>
- </property>
-
- <property>
- <name>hbase.coprocessor.master.classes</name>
- <value></value>
- <description>A comma-separated list of
- org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are
- loaded by default on the active HMaster process. For any implemented
- coprocessor methods, the listed classes will be called in order. After
- implementing your own MasterObserver, just put it in HBase's classpath
- and add the fully qualified class name here.
- </description>
- </property>
-
- <property>
- <name>hbase.zookeeper.property.clientPort</name>
- <value>2181</value>
- <description>Property from ZooKeeper's config zoo.cfg.
- The port at which the clients will connect.
- </description>
- </property>
-
- <!--
- The following three properties are used together to create the list of
- host:peer_port:leader_port quorum servers for ZooKeeper.
- -->
- <property>
- <name>hbase.zookeeper.quorum</name>
- <value>localhost</value>
- <description>Comma separated list of servers in the ZooKeeper Quorum.
- For example, "host1.mydomain.com,host2.mydomain.com,host3.mydomain.com".
- By default this is set to localhost for local and pseudo-distributed modes
- of operation. For a fully-distributed setup, this should be set to a full
- list of ZooKeeper quorum servers. If HBASE_MANAGES_ZK is set in hbase-env.sh
- this is the list of servers which we will start/stop ZooKeeper on.
- </description>
- </property>
- <!-- End of properties used to generate ZooKeeper host:port quorum list. -->
-
- <property>
- <name>hbase.zookeeper.useMulti</name>
- <value>true</value>
- <description>Instructs HBase to make use of ZooKeeper's multi-update functionality.
- This allows certain ZooKeeper operations to complete more quickly and prevents some issues
- with rare Replication failure scenarios (see the release note of HBASE-2611 for an example).·
- IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+
- and will not be downgraded. ZooKeeper versions before 3.4 do not support multi-update and will
- not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495).
- </description>
- </property>
- <property>
- <name>zookeeper.znode.parent</name>
- <value>/hbase-unsecure</value>
- <description>Root ZNode for HBase in ZooKeeper. All of HBase's ZooKeeper
- files that are configured with a relative path will go under this node.
- By default, all of HBase's ZooKeeper file path are configured with a
- relative path, so they will all go under this directory unless changed.
- </description>
- </property>
-
- <property>
- <name>hbase.defaults.for.version.skip</name>
- <value>true</value>
- <description>Disables version verification.</description>
- </property>
-
- <property>
- <name>dfs.domain.socket.path</name>
- <value>/var/lib/hadoop-hdfs/dn_socket</value>
- <description>Path to domain socket.</description>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/ganglia_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/ganglia_metrics.json b/app-packages/hbase-v0_96/ganglia_metrics.json
deleted file mode 100644
index da73d48..0000000
--- a/app-packages/hbase-v0_96/ganglia_metrics.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "Component": {
- "HBASE_REGIONSERVER": {
- "readRequestsCount": {
- "metric": "regionserver.Server.readRequestCount",
- "pointInTime": false,
- "temporal": true
- },
- "regions": {
- "metric": "regionserver.Server.regionCount",
- "pointInTime": false,
- "temporal": true
- },
- "flushQueueSize": {
- "metric": "regionserver.Server.flushQueueLength",
- "pointInTime": false,
- "temporal": true
- }
- },
- "HBASE_MASTER": {
- "cluster_requests": {
- "metric": "master.Server.clusterRequests",
- "pointInTime": false,
- "temporal": true
- },
- "splitTime_avg_time": {
- "metric": "master.FileSystem.HlogSplitTime_mean",
- "pointInTime": false,
- "temporal": true
- },
- "splitSize_avg_time": {
- "metric": "master.FileSystem.HlogSplitSize_mean",
- "pointInTime": false,
- "temporal": true
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/jmx_metrics.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/jmx_metrics.json b/app-packages/hbase-v0_96/jmx_metrics.json
deleted file mode 100644
index ac0640e..0000000
--- a/app-packages/hbase-v0_96/jmx_metrics.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
- "Component": {
- "HBASE_MASTER": {
- "MetricAverageLoad": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.averageLoad",
- "pointInTime": true,
- "temporal": false
- },
- "DeadRegionServers": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.numDeadRegionServers",
- "pointInTime": true,
- "temporal": false
- },
- "ClusterId": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.clusterId",
- "pointInTime": true,
- "temporal": false
- },
- "IsActiveMaster": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.isActiveMaster",
- "pointInTime": true,
- "temporal": false
- },
- "MasterActiveTime": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.masterActiveTime",
- "pointInTime": true,
- "temporal": false
- },
- "MasterStartTime": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.masterStartTime",
- "pointInTime": true,
- "temporal": false
- },
- "RegionServers": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.numRegionServers",
- "pointInTime": true,
- "temporal": false
- },
- "ServerName": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.serverName",
- "pointInTime": true,
- "temporal": false
- },
- "ZookeeperQuorum": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.tag.zookeeperQuorum",
- "pointInTime": true,
- "temporal": false
- },
- "ClusterRequests": {
- "metric": "Hadoop:service=HBase,name=Master,sub=Server.clusterRequests",
- "pointInTime": true,
- "temporal": false
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/metainfo.xml b/app-packages/hbase-v0_96/metainfo.xml
deleted file mode 100644
index 7eb6867..0000000
--- a/app-packages/hbase-v0_96/metainfo.xml
+++ /dev/null
@@ -1,109 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>HBASE</name>
- <comment>
- Apache HBase is the Hadoop database, a distributed, scalable, big data store.
- Requirements:
- 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner.
- 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance.
- </comment>
- <version>0.96.0.2.1.1</version>
- <type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
- <exportGroups>
- <exportGroup>
- <name>QuickLinks</name>
- <exports>
- <export>
- <name>org.apache.slider.jmx</name>
- <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
- </export>
- <export>
- <name>org.apache.slider.monitor</name>
- <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
- </export>
- <export>
- <name>org.apache.slider.metrics</name>
- <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
- </export>
- <export>
- <name>org.apache.slider.ganglia</name>
- <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
- </export>
- </exports>
- </exportGroup>
- </exportGroups>
- <commandOrders>
- <commandOrder>
- <command>HBASE_REGIONSERVER-START</command>
- <requires>HBASE_MASTER-STARTED</requires>
- </commandOrder>
- </commandOrders>
- <components>
- <component>
- <name>HBASE_MASTER</name>
- <category>MASTER</category>
- <minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
- <commandScript>
- <script>scripts/hbase_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
-
- <component>
- <name>HBASE_REGIONSERVER</name>
- <category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
- <commandScript>
- <script>scripts/hbase_regionserver.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
-
- <component>
- <name>HBASE_CLIENT</name>
- <category>CLIENT</category>
- <minInstanceCount>0</minInstanceCount>
- <commandScript>
- <script>scripts/hbase_client.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/hbase-0.96.1-hadoop2-bin.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE b/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
deleted file mode 100644
index 5d03caa..0000000
--- a/app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Replace with actual hbase tarball.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/__init__.py b/app-packages/hbase-v0_96/package/scripts/__init__.py
deleted file mode 100644
index 5561e10..0000000
--- a/app-packages/hbase-v0_96/package/scripts/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/functions.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/functions.py b/app-packages/hbase-v0_96/package/scripts/functions.py
deleted file mode 100644
index e6e7fb9..0000000
--- a/app-packages/hbase-v0_96/package/scripts/functions.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import os
-import re
-import math
-import datetime
-
-from resource_management.core.shell import checked_call
-
-def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
- """
- @param heapsize_str: str (e.g '1000m')
- @param xmn_percent: float (e.g 0.2)
- @param xmn_max: integer (e.g 512)
- """
- heapsize = int(re.search('\d+',heapsize_str).group(0))
- heapsize_unit = re.search('\D+',heapsize_str).group(0)
- xmn_val = int(math.floor(heapsize*xmn_percent))
- xmn_val -= xmn_val % 8
-
- result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
- return str(result_xmn_val) + heapsize_unit
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase.py b/app-packages/hbase-v0_96/package/scripts/hbase.py
deleted file mode 100644
index ed6ec51..0000000
--- a/app-packages/hbase-v0_96/package/scripts/hbase.py
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-import os
-
-from resource_management import *
-import sys
-
-def hbase(name=None # 'master' or 'regionserver' or 'client'
- ):
- import params
-
- if name in ["master","regionserver"]:
- params.HdfsDirectory(params.hbase_hdfs_root_dir,
- action="create_delayed"
- )
- params.HdfsDirectory(params.hbase_staging_dir,
- action="create_delayed",
- mode=0711
- )
- params.HdfsDirectory(None, action="create")
- Directory( params.conf_dir,
- owner = params.hbase_user,
- group = params.user_group,
- recursive = True
- )
-
- Directory (params.tmp_dir,
- owner = params.hbase_user,
- recursive = True
- )
-
- Directory (os.path.join(params.local_dir, "jars"),
- owner = params.hbase_user,
- group = params.user_group,
- mode=0775,
- recursive = True
- )
-
- XmlConfig( "hbase-site.xml",
- conf_dir = params.conf_dir,
- configurations = params.config['configurations']['hbase-site'],
- owner = params.hbase_user,
- group = params.user_group
- )
-
- XmlConfig( "hdfs-site.xml",
- conf_dir = params.conf_dir,
- configurations = params.config['configurations']['hdfs-site'],
- owner = params.hbase_user,
- group = params.user_group
- )
-
- if 'hbase-policy' in params.config['configurations']:
- XmlConfig( "hbase-policy.xml",
- configurations = params.config['configurations']['hbase-policy'],
- owner = params.hbase_user,
- group = params.user_group
- )
- # Manually overriding ownership of file installed by hadoop package
- else:
- File( format("{conf_dir}/hbase-policy.xml"),
- owner = params.hbase_user,
- group = params.user_group
- )
-
- hbase_TemplateConfig( 'hbase-env.sh')
-
- hbase_TemplateConfig( params.metric_prop_file_name,
- tag = 'GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS'
- )
-
- if params.security_enabled:
- hbase_TemplateConfig( format("hbase_{name}_jaas.conf"))
-
- if name != "client":
- Directory( params.pid_dir,
- owner = params.hbase_user,
- recursive = True
- )
-
- Directory (params.log_dir,
- owner = params.hbase_user,
- recursive = True
- )
-
- if (params.log4j_props != None):
- File(format("{params.conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hbase_user,
- content=params.log4j_props
- )
- elif (os.path.exists(format("{params.conf_dir}/log4j.properties"))):
- File(format("{params.conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hbase_user
- )
-
-def hbase_TemplateConfig(name,
- tag=None
- ):
- import params
-
- TemplateConfig( format("{conf_dir}/{name}"),
- owner = params.hbase_user,
- template_tag = tag
- )
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/hbase_client.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_client.py b/app-packages/hbase-v0_96/package/scripts/hbase_client.py
deleted file mode 100644
index 043ad11..0000000
--- a/app-packages/hbase-v0_96/package/scripts/hbase_client.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-
-from hbase import hbase
-
-
-class HbaseClient(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
-
- hbase(name='client')
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-
-if __name__ == "__main__":
- HbaseClient().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/hbase_master.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_master.py b/app-packages/hbase-v0_96/package/scripts/hbase_master.py
deleted file mode 100644
index 47b2409..0000000
--- a/app-packages/hbase-v0_96/package/scripts/hbase_master.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-
-from hbase import hbase
-from hbase_service import hbase_service
-
-
-class HbaseMaster(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
-
- hbase(name='master')
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env) # for security
-
- hbase_service( 'master',
- action = 'start'
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- hbase_service( 'master',
- action = 'stop'
- )
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- pid_file = format("{pid_dir}/hbase-{hbase_user}-master.pid")
- check_process_status(pid_file)
-
-
-if __name__ == "__main__":
- HbaseMaster().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py b/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py
deleted file mode 100644
index 8d66dcc..0000000
--- a/app-packages/hbase-v0_96/package/scripts/hbase_regionserver.py
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-
-from hbase import hbase
-from hbase_service import hbase_service
-
-
-class HbaseRegionServer(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
-
- hbase(name='regionserver')
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env) # for security
-
- hbase_service( 'regionserver',
- action = 'start'
- )
-
- def stop(self, env):
- import params
- env.set_params(params)
-
- hbase_service( 'regionserver',
- action = 'stop'
- )
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
- check_process_status(pid_file)
-
- def decommission(self, env):
- print "Decommission not yet implemented!"
-
-
-if __name__ == "__main__":
- HbaseRegionServer().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/hbase_service.py b/app-packages/hbase-v0_96/package/scripts/hbase_service.py
deleted file mode 100644
index 2b30083..0000000
--- a/app-packages/hbase-v0_96/package/scripts/hbase_service.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-def hbase_service(
- name,
- action = 'start'): # 'start' or 'stop' or 'status'
-
- import params
-
- role = name
- cmd = format("{daemon_script} --config {conf_dir}")
- pid_file = format("{pid_dir}/hbase-{hbase_user}-{role}.pid")
-
- daemon_cmd = None
- no_op_test = None
-
- if action == 'start':
- daemon_cmd = format("{cmd} start {role}")
- no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
- elif action == 'stop':
- daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
-
- if daemon_cmd is not None:
- Execute ( daemon_cmd,
- not_if = no_op_test
- )
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/params.py b/app-packages/hbase-v0_96/package/scripts/params.py
deleted file mode 100644
index 0d8b04b..0000000
--- a/app-packages/hbase-v0_96/package/scripts/params.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from functions import calc_xmn_from_xms
-from resource_management import *
-import status_params
-
-# server configurations
-config = Script.get_config()
-
-hbase_root = config['configurations']['global']['app_root']
-conf_dir = format("{hbase_root}/conf")
-daemon_script = format("{hbase_root}/bin/hbase-daemon.sh")
-
-hbase_user = status_params.hbase_user
-_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
-user_group = config['configurations']['global']['user_group']
-
-# this is "hadoop-metrics.properties" for 1.x stacks
-metric_prop_file_name = "hadoop-metrics2-hbase.properties"
-
-# not supporting 32 bit jdk.
-java64_home = config['hostLevelParams']['java_home']
-
-log_dir = config['configurations']['global']['app_log_dir']
-master_heapsize = config['configurations']['global']['hbase_master_heapsize']
-
-regionserver_heapsize = config['configurations']['global']['hbase_regionserver_heapsize']
-regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
-
-pid_dir = status_params.pid_dir
-tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
-local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site'])
-
-client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf"))
-master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))
-regionserver_jaas_config_file = default('hbase_regionserver_jaas_config_file', format("{conf_dir}/hbase_regionserver_jaas.conf"))
-
-ganglia_server_host = default('/configurations/global/ganglia_server_host', '')
-ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663')
-
-if security_enabled:
-
- _use_hostname_in_principal = default('instance_name', True)
- _master_primary_name = config['configurations']['global']['hbase_master_primary_name']
- _hostname_lowercase = config['hostname'].lower()
- _kerberos_domain = config['configurations']['global']['kerberos_domain']
- _master_principal_name = config['configurations']['global']['hbase_master_principal_name']
- _regionserver_primary_name = config['configurations']['global']['hbase_regionserver_primary_name']
-
- if _use_hostname_in_principal:
- master_jaas_princ = format("{_master_primary_name}/{_hostname_lowercase}@{_kerberos_domain}")
- regionserver_jaas_princ = format("{_regionserver_primary_name}/{_hostname_lowercase}@{_kerberos_domain}")
- else:
- master_jaas_princ = format("{_master_principal_name}@{_kerberos_domain}")
- regionserver_jaas_princ = format("{_regionserver_primary_name}@{_kerberos_domain}")
-
-master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
-regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
-kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-if security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};")
-else:
- kinit_cmd = ""
-
-#log4j.properties
-if (('hbase-log4j' in config['configurations']) and ('content' in config['configurations']['hbase-log4j'])):
- log4j_props = config['configurations']['hbase-log4j']['content']
-else:
- log4j_props = None
-
-
-hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
-hbase_staging_dir = config['configurations']['hbase-site']['hbase.stagingdir']
-#for create_hdfs_directory
-hostname = config["hostname"]
-hadoop_conf_dir = "/etc/hadoop/conf"
-hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab']
-hdfs_user = config['configurations']['global']['hdfs_user']
-kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local
-)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/scripts/status_params.py b/app-packages/hbase-v0_96/package/scripts/status_params.py
deleted file mode 100644
index c18cbb9..0000000
--- a/app-packages/hbase-v0_96/package/scripts/status_params.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-
-config = Script.get_config()
-
-pid_dir = config['configurations']['global']['app_pid_dir']
-hbase_user = config['configurations']['global']['app_user']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2 b/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
deleted file mode 100644
index a3eb1fc..0000000
--- a/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# See http://wiki.apache.org/hadoop/GangliaMetrics
-#
-# Make sure you know whether you are using ganglia 3.0 or 3.1.
-# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
-# And, yes, this file is named hadoop-metrics.properties rather than
-# hbase-metrics.properties because we're leveraging the hadoop metrics
-# package and hadoop-metrics.properties is an hardcoded-name, at least
-# for the moment.
-#
-# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
-
-# HBase-specific configuration to reset long-running stats (e.g. compactions)
-# If this variable is left out, then the default is no expiration.
-hbase.extendedperiod = 3600
-
-# Configuration of the "hbase" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-hbase.period=10
-hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}}
-
-# Configuration of the "jvm" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-jvm.period=10
-jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}}
-
-# Configuration of the "rpc" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-rpc.period=10
-rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}}
-
-#Ganglia following hadoop example
-hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
-hbase.sink.ganglia.period=10
-
-# default for supportsparse is false
-*.sink.ganglia.supportsparse=true
-
-.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
-.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-
-hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2 b/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
deleted file mode 100644
index a3eb1fc..0000000
--- a/app-packages/hbase-v0_96/package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+++ /dev/null
@@ -1,62 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# See http://wiki.apache.org/hadoop/GangliaMetrics
-#
-# Make sure you know whether you are using ganglia 3.0 or 3.1.
-# If 3.1, you will have to patch your hadoop instance with HADOOP-4675
-# And, yes, this file is named hadoop-metrics.properties rather than
-# hbase-metrics.properties because we're leveraging the hadoop metrics
-# package and hadoop-metrics.properties is an hardcoded-name, at least
-# for the moment.
-#
-# See also http://hadoop.apache.org/hbase/docs/current/metrics.html
-
-# HBase-specific configuration to reset long-running stats (e.g. compactions)
-# If this variable is left out, then the default is no expiration.
-hbase.extendedperiod = 3600
-
-# Configuration of the "hbase" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-hbase.period=10
-hbase.servers={{ganglia_server_host}}:{{ganglia_server_port}}
-
-# Configuration of the "jvm" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-jvm.period=10
-jvm.servers={{ganglia_server_host}}:{{ganglia_server_port}}
-
-# Configuration of the "rpc" context for ganglia
-# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter)
-# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
-rpc.period=10
-rpc.servers={{ganglia_server_host}}:{{ganglia_server_port}}
-
-#Ganglia following hadoop example
-hbase.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
-hbase.sink.ganglia.period=10
-
-# default for supportsparse is false
-*.sink.ganglia.supportsparse=true
-
-.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
-.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-
-hbase.sink.ganglia.servers={{ganglia_server_host}}:{{ganglia_server_port}}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2 b/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2
deleted file mode 100644
index 4aa79ad..0000000
--- a/app-packages/hbase-v0_96/package/templates/hbase-env.sh.j2
+++ /dev/null
@@ -1,81 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set environment variables here.
-
-# The java implementation to use. Java 1.6 required.
-export JAVA_HOME={{java64_home}}
-
-# HBase Configuration directory
-export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{conf_dir}}}
-
-# Extra Java CLASSPATH elements. Optional.
-export HBASE_CLASSPATH=${HBASE_CLASSPATH}
-
-# The maximum amount of heap to use, in MB. Default is 1000.
-# export HBASE_HEAPSIZE=1000
-
-# Extra Java runtime options.
-# Below are what we set by default. May only work with SUN JVM.
-# For more on why as well as other possible settings,
-# see http://wiki.apache.org/hadoop/PerformanceTuning
-export HBASE_OPTS="-XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log"
-export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`"
-# Uncomment below to enable java garbage collection logging.
-# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
-
-# Uncomment and adjust to enable JMX exporting
-# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
-# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-#
-# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
-export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
-export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
-# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
-# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
-
-# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
-export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
-
-# Extra ssh options. Empty by default.
-# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
-
-# Where log files are stored. $HBASE_HOME/logs by default.
-export HBASE_LOG_DIR={{log_dir}}
-
-# A string representing this instance of hbase. $USER by default.
-# export HBASE_IDENT_STRING=$USER
-
-# The scheduling priority for daemon processes. See 'man nice'.
-# export HBASE_NICENESS=10
-
-# The directory where pid files are stored. /tmp by default.
-export HBASE_PID_DIR={{pid_dir}}
-
-# Seconds to sleep between slave commands. Unset by default. This
-# can be useful in large clusters, where, e.g., slave rsyncs can
-# otherwise arrive faster than the master can service them.
-# export HBASE_SLAVE_SLEEP=0.1
-
-# Tell HBase whether it should manage it's own instance of Zookeeper or not.
-export HBASE_MANAGES_ZK=false
-
-{% if security_enabled %}
-export HBASE_OPTS="$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}"
-export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}}"
-export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}}"
-{% endif %}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2 b/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2
deleted file mode 100644
index bb4279c..0000000
--- a/app-packages/hbase-v0_96/package/templates/hbase_client_jaas.conf.j2
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-Client {
-com.sun.security.auth.module.Krb5LoginModule required
-useKeyTab=false
-useTicketCache=true;
-};
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2 b/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2
deleted file mode 100644
index 91ce3ef..0000000
--- a/app-packages/hbase-v0_96/package/templates/hbase_master_jaas.conf.j2
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-Client {
-com.sun.security.auth.module.Krb5LoginModule required
-useKeyTab=true
-storeKey=true
-useTicketCache=false
-keyTab="{{master_keytab_path}}"
-principal="{{master_jaas_princ}}";
-};
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2 b/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2
deleted file mode 100644
index 2a9b9f3..0000000
--- a/app-packages/hbase-v0_96/package/templates/hbase_regionserver_jaas.conf.j2
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-Client {
-com.sun.security.auth.module.Krb5LoginModule required
-useKeyTab=true
-storeKey=true
-useTicketCache=false
-keyTab="{{regionserver_keytab_path}}"
-principal="{{regionserver_jaas_princ}}";
-};
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/package/templates/regionservers.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/package/templates/regionservers.j2 b/app-packages/hbase-v0_96/package/templates/regionservers.j2
deleted file mode 100644
index 81d060b..0000000
--- a/app-packages/hbase-v0_96/package/templates/regionservers.j2
+++ /dev/null
@@ -1,20 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-{% for host in rs_hosts %}{{host}}
-{% endfor %}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase-v0_96/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/resources.json b/app-packages/hbase-v0_96/resources.json
deleted file mode 100644
index 9cc1b47..0000000
--- a/app-packages/hbase-v0_96/resources.json
+++ /dev/null
@@ -1,19 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- },
- "components": {
- "HBASE_MASTER": {
- "yarn.role.priority": "1",
- "yarn.component.instances": "1"
- },
- "slider-appmaster": {
- },
- "HBASE_REGIONSERVER": {
- "yarn.role.priority": "2",
- "yarn.component.instances": "1"
- }
- }
-}
\ No newline at end of file
[23/50] [abbrv] SLIDER-121 removed site documentation from git source
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/core.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/core.md b/src/site/markdown/configuration/core.md
deleted file mode 100644
index 319bf77..0000000
--- a/src/site/markdown/configuration/core.md
+++ /dev/null
@@ -1,407 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Core Configuration Specification
-
-
-## Terminology
-
-
-*Application* A single application, such as an HBase cluster. An application
-is distribed across the YARN cluster.
-
-*Component* A single executable part of the larger application. An application
-may have multiple components, and multiple instances of each component.
-
-*YARN* Yet Another Resource Negotiator
-
-*YARN Resource Requirements* The requirements for a YARN resource request.
-Currently this consists of RAM and CPU requirements.
-
-*YARN Container*. An allocation portion of a servers resources granted
-to satisfy the requested YARN resource requirements. A process can be deployed
-to a container.
-
-
-*`resources.json`*: A file that describes the
-size of the application in terms of its component requirements: how many,
-and what their resource requirements are.
-
-*`application.json`*: A file that describes the
-size of the application in terms of its component requirements: how many,
-and what their resource requirements are.
-
-## Structure
-
-Configurations are stored in well-formed JSON files.
-1. Text MUST be saved in the UTF-8 format.
-1. Duplicate entries MUST NOT occur in any section.
-1. The ordering of elements is NOT significant.
-
-The JSON specification files all have a similar structure
-
-1. A `schema` string indicating version. Currently this is temporarily set to
-
- "http://example.org/specification/v2.0.0"
-
-
-1. A global section, `/global` containing string properties
-1. A component section, `/components`.
-1. 0 or more sections under `/components` for each component, identified by component name,
- containing string properties.
-1. 0 or 1 section `/metadata` containing arbitrary metadata (such as a description,
-author, or any other information that is not parsed or processed directly).
-
-
-The simplest valid specification file is
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- },
- "components": {
- }
- }
-
-
-## Property inheritance model and *resolution*
-
-
-There is a simple global to component inheritance model.
-
-1. Properties defined in `/global` define parameters across the entire application.
-1. Properties defined a section under `/components` define parameters for
-a specific component in the application.
-1. All global properties are propagated to each component.
-1. A component section may override any global property.
-1. The final set of configuration properties for a component is the global
-properties extended and overridden by the global set.
-1. The process of expanding the properties is termed *resolution*; the *resolved*
-specification is the outcome.
-1. There is NO form of explicitly cross-referencing another attribute. This
-MAY be added in future.
-1. There is NO sharing of information from the different `.json` files in a
-an application configuration.
-
-### Example
-
-Here is an example configuration
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- "g1": "a",
- "g2": "b"
- },
- "components": {
- "simple": {
- },
- "master": {
- "name": "m",
- "g1": "overridden"
-
- },
- "worker": {
- "name": "w",
- "g1": "overridden-by-worker",
- "timeout": "1000"
- }
- }
- }
-
-The `/global` section defines two properties
-
- g1="a"
- g2="b"
-
-These are the values visible to any part of the application which is
-not itself one of the components.
-
-
-There are three components defined, `simple`, `master` and `worker`.
-
-
-#### component `simple`:
-
- g1="a"
- g2="b"
-
-
-No settings have been defined specifically for the component; the global
-settings are applied.
-
-#### component `master`:
-
- name="m",
- g1="overridden"
- g2="b"
-
-A new attribute, `name`, has been defined with the value `"m"`, and the
-global property `g1` has been overridden with the new value, `"overridden"`.
-The global property `g2` is passed down unchanged.
-
-
-#### component `worker`:
-
- name="w",
- g1="overridden-by-worker"
- g2="b"
- timeout: "1000"
-
-A new attribute, `name`, has been defined with the value `"w"`, and another,
-`timeout`, value "1000".
-
-The global property `g1` has been overridden with the new value, `"overridden-by-worker"`.
-
-The global property `g2` is passed down unchanged.
-
-This example shows some key points about the design
-
-* each component gets its own map of properties, which is independent from
- that of other components.
-* all global properties are either present or overridden by a new value.
- They can not be "undefined"
-* new properties defined in a component are not visible to any other component.
-
-The final *resolved* model is as follows
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- "g1": "a",
- "g2": "b"
- },
- "components": {
- "simple": {
- "g1": "a",
- "g2": "b"
- },
- "master": {
- "name": "m",
- "g1": "overridden",
- "g2": "b"
- },
- "worker": {
- "name": "m",
- "g1": "overridden-by-worker",
- "g2": "b",
- "timeout": "1000"
- }
- }
- }
-
-This the specification JSON that would have generate exactly the same result as
-in the example, without any propagation of data from the global section
-to individual components.
-
-Note that a resolved specification can still have the resolution operation applied
-to it -it just does not have any effect.
-
-## Metadata
-
-The metadata section can contain arbitrary string values for use in diagnostics
-and by other applications.
-
-To avoid conflict with other applications, please use a unique name in strings,
-such as java-style package names.
-
-# Resource Requirements: `resources.json`
-
-This file declares the resource requirements for YARN for the components
-of an application.
-
-`instances`: the number of instances of a role desired.
-`yarn.vcores`: number of "virtual" required by a component.
-`yarn.memory`: the number of megabytes required by a component.
-
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- "description": "example of a resources file"
- },
-
- "global": {
- "yarn.vcores": "1",
- "yarn.memory": "512"
- },
-
- "components": {
- "master": {
- "instances": "1",
- "yarn.memory": "1024"
- },
- "worker": {
- "instances":"5"
- }
- }
- }
-
-The resolved file would be
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- "description": "example of a resources file"
- },
-
- "global": {
- "yarn.vcores": "1",
- "yarn.memory": "512"
- },
-
- "components": {
- "master": {
- "instances": "1",
- "yarn.vcores": "1",
- "yarn.memory": "1024"
- },
- "worker": {
- "instances":"5",
- "yarn.vcores": "1",
- "yarn.memory": "512"
- }
- }
- }
-
-This declares this deployment of the application to consist of one instance of
-the master component, using 1 vcore and 1024MB of RAM, and five worker components
-each using one vcore and 512 MB of RAM.
-
-
-## Internal information, `internal.json`
-
-This contains internal data related to the deployment -it is not
-intended for manual editing.
-
-There MAY be a component, `diagnostics`. If defined, its content contains
-diagnostic information for support calls, and MUST NOT be interpreted
-during application deployment, (though it may be included in the generation
-of diagnostics reports)
-
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- "description": "Internal configuration DO NOT EDIT"
- },
- "global": {
- "name": "small_cluster",
- "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
- },
- "components": {
-
- "diagnostics": {
- "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
- "create.hadoop.build.info": "2.3.0",
- "create.time.millis": "1393512091276",
- "create.time": "27 Feb 2014 14:41:31 GMT"
- }
- }
- }
-
-
-## Deployment specification: `app_configuration.json`
-
-
-This defines parameters that are to be used when creating the instance of the
-application, and instances of the individual components.
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
-
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- },
- "components": {
-
- "worker": {
- "jvm.heapsize": "512M"
- },
- "master": {
- "jvm.heapsize": "512M"
- }
- }
- }
-
-The resolved specification defines the values that are passed to the
-different components.
-
- {
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- },
- "components": {
-
- "worker": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- },
- "master": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- }
- }
- }
-
-The `site.` properties have been passed down to each component, components
-whose templates may generate local site configurations. The override model
-does not prevent any component from overriding global configuration so as
-to create local configurations incompatible with the global state. (i.e.,
-there is no way to declare an attribute as final). It is the responsibility
-of the author of the configuration file (and their tools) to detect such issues.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-app_configuration-resolved.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-app_configuration-resolved.json b/src/site/markdown/configuration/example-app_configuration-resolved.json
deleted file mode 100644
index 5b90ba9..0000000
--- a/src/site/markdown/configuration/example-app_configuration-resolved.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- },
- "components": {
-
- "worker": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- },
- "master": {
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0",
- "jvm.heapsize": "512M"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-app_configuration.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-app_configuration.json b/src/site/markdown/configuration/example-app_configuration.json
deleted file mode 100644
index 489acda..0000000
--- a/src/site/markdown/configuration/example-app_configuration.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
-
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_small_cluster",
- "zookeeper.hosts": "zoo1,zoo2,zoo3",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://cluster:8020",
- "site.fs.default.name": "hdfs://cluster:8020",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- },
- "components": {
-
- "worker": {
- "jvm.heapsize": "512M"
- },
- "master": {
- "jvm.heapsize": "512M"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-empty.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-empty.json b/src/site/markdown/configuration/example-empty.json
deleted file mode 100644
index 5c05163..0000000
--- a/src/site/markdown/configuration/example-empty.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- },
- "components": {
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-internal.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-internal.json b/src/site/markdown/configuration/example-internal.json
deleted file mode 100644
index 8617d1f..0000000
--- a/src/site/markdown/configuration/example-internal.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- "description": "Internal configuration DO NOT EDIT"
- },
- "global": {
- "application.name": "small_cluster",
- "application.type": "hbase",
- "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
- },
- "components": {
-
- "diagnostics": {
- "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
- "create.hadoop.build.info": "2.3.0",
- "create.time.millis": "1393512091276",
- "create.time": "27 Feb 2014 14:41:31 GMT"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-overridden-resolved.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-overridden-resolved.json b/src/site/markdown/configuration/example-overridden-resolved.json
deleted file mode 100644
index 2b810b5..0000000
--- a/src/site/markdown/configuration/example-overridden-resolved.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- "g1": "a",
- "g2": "b"
- },
- "components": {
- "simple": {
- "g1": "a",
- "g2": "b"
- },
- "master": {
- "name": "m",
- "g1": "overridden",
- "g2": "b"
- },
- "worker": {
- "name": "m",
- "g1": "overridden-by-worker",
- "g2": "b",
- "timeout": "1000"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-overridden.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-overridden.json b/src/site/markdown/configuration/example-overridden.json
deleted file mode 100644
index 9a74143..0000000
--- a/src/site/markdown/configuration/example-overridden.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "global": {
- "g1": "a",
- "g2": "b"
- },
- "components": {
- "simple": {
- },
- "master": {
- "name": "m",
- "g1": "overridden"
-
- },
- "worker": {
- "name": "m",
- "g1": "overridden-by-worker",
- "timeout": "1000"
-
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/example-resources.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/example-resources.json b/src/site/markdown/configuration/example-resources.json
deleted file mode 100644
index 06c3b54..0000000
--- a/src/site/markdown/configuration/example-resources.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- "description": "example of a resources file"
- },
-
- "global": {
- "yarn.vcores": "1",
- "yarn.memory": "512"
- },
-
- "components": {
- "master": {
- "instances": "1",
- "yarn.vcores": "1",
- "yarn.memory": "1024"
- },
- "worker": {
- "instances":"5",
- "yarn.vcores": "1",
- "yarn.memory": "512"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/index-markdown.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/index-markdown.md b/src/site/markdown/configuration/index-markdown.md
deleted file mode 100644
index b5a2186..0000000
--- a/src/site/markdown/configuration/index-markdown.md
+++ /dev/null
@@ -1,31 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: Specification of an application instance, revision 2.0
-
-The specification of an applicaton, comprises
-1. The persistent description of an application's configuration
-1. The persistent description of the desired topology and YARN resource
-requirements.
-1. The dynamic description of the running application, including information
-on the location of components and aggregated statistics.
-
-
-1. [Redesign](redesign.md)
-1. [Specification](specification.md)
-1. [Example: current](original-hbase.json)
-1. [Example: proposed](proposed-hbase.json)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/index.md b/src/site/markdown/configuration/index.md
deleted file mode 100644
index 160f84d..0000000
--- a/src/site/markdown/configuration/index.md
+++ /dev/null
@@ -1,38 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: Specification of an application instance, revision 2.0
-
-The specification of an applicaton, comprises
-1. The persistent description of an application's configuration
-1. The persistent description of the desired topology and YARN resource
-requirements.
-1. The dynamic description of the running application, including information
-on the location of components and aggregated statistics.
-
-The specifics of this are covered in the [Core Configuration Specification](core.md)
-
-
-## Historical References
-
-1. [Specification](specification.html)
-1. [Redesign](redesign.html)
-
-
-1. [Example: current](original-hbase.json)
-1. [Example: proposed](proposed-hbase.json)
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/original-hbase.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/original-hbase.json b/src/site/markdown/configuration/original-hbase.json
deleted file mode 100644
index 257caeb..0000000
--- a/src/site/markdown/configuration/original-hbase.json
+++ /dev/null
@@ -1,139 +0,0 @@
-{
- "version": "1.0",
- "name": "test_cluster_lifecycle",
- "type": "hbase",
- "state": 3,
- "createTime": 1393512091276,
- "updateTime": 1393512117286,
- "originConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot",
- "generatedConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/generated",
- "dataPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database",
- "options": {
- "zookeeper.port": "2181",
- "site.hbase.master.startup.retainassign": "true",
- "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz",
- "site.fs.defaultFS": "hdfs://sandbox:8020",
- "slider.container.failure.threshold": "5",
- "site.fs.default.name": "hdfs://sandbox:8020",
- "slider.cluster.directory.permissions": "0770",
- "slider.am.monitoring.enabled": "false",
- "zookeeper.path": "/yarnapps_slider_stevel_test_cluster_lifecycle",
- "slider.tmp.dir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/tmp/am",
- "slider.data.directory.permissions": "0770",
- "zookeeper.hosts": "sandbox",
- "slider.container.failure.shortlife": "60"
- },
- "info": {
- "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
- "create.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
- "create.hadoop.build.info": "2.3.0",
- "create.time.millis": "1393512091276",
- "create.time": "27 Feb 2014 14:41:31 GMT",
- "slider.am.restart.supported": "false",
- "live.time": "27 Feb 2014 14:41:56 GMT",
- "live.time.millis": "1393512116881",
- "status.time": "27 Feb 2014 14:42:08 GMT",
- "status.time.millis": "1393512128726",
- "yarn.vcores": "32",
- "yarn.memory": "2048",
- "status.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
- "status.hadoop.build.info": "2.3.0",
- "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
- },
- "statistics": {
- "worker": {
- "containers.start.started": 0,
- "containers.live": 0,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 0,
- "containers.requested": 0
- },
- "slider": {
- "containers.unknown.completed": 0,
- "containers.start.started": 0,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.surplus": 0
- },
- "master": {
- "containers.start.started": 0,
- "containers.live": 0,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 0,
- "containers.requested": 0
- }
- },
- "status": {
- },
- "instances": {
- "slider": [ "container_1393511571284_0002_01_000001" ]
- },
- "roles": {
- "worker": {
- "yarn.memory": "768",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "0",
- "role.requested.instances": "0",
- "role.name": "worker",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "0",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1",
- "role.releasing.instances": "0",
- "role.failed.instances": "0",
- "app.infoport": "0"
- },
- "slider": {
- "yarn.memory": "256",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "1",
- "role.requested.instances": "0",
- "role.name": "slider",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "1",
- "jvm.heapsize": "256M",
- "yarn.vcores": "1",
- "role.releasing.instances": "0",
- "role.failed.instances": "0"
- },
- "master": {
- "yarn.memory": "1024",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "0",
- "role.requested.instances": "0",
- "role.name": "master",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "0",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1",
- "role.releasing.instances": "0",
- "role.failed.instances": "0",
- "app.infoport": "0"
- }
- },
- "clientProperties": {
- "fs.defaultFS": "hdfs://sandbox:8020",
- "hbase.cluster.distributed": "true",
- "hbase.master.info.port": "0",
- "hbase.master.port": "0",
- "hbase.master.startup.retainassign": "true",
- "hbase.regionserver.hlog.tolerable.lowreplication": "1",
- "hbase.regionserver.info.port": "0",
- "hbase.regionserver.port": "0",
- "hbase.rootdir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database",
- "hbase.tmp.dir": "./hbase-tmp",
- "hbase.zookeeper.property.clientPort": "2181",
- "hbase.zookeeper.quorum": "sandbox",
- "slider.template.origin": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot/hbase-site.xml",
- "slider.unused.option": "1",
- "zookeeper.znode.parent": "/yarnapps_slider_stevel_test_cluster_lifecycle"
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/proposed-hbase.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/proposed-hbase.json b/src/site/markdown/configuration/proposed-hbase.json
deleted file mode 100644
index c4f637f..0000000
--- a/src/site/markdown/configuration/proposed-hbase.json
+++ /dev/null
@@ -1,273 +0,0 @@
-{
- "version": "2.0.0",
- "name": "test_cluster_lifecycle",
- "valid`": true,
-
- "slider-internal":{
- "type": "hbase",
- "createTime": 1393512091276,
- "updateTime": 1393512117286,
- "originConfigurationPath": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/snapshot",
- "generatedConfigurationPath": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/generated",
- "dataPath": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/database",
- "slider.tmp.dir": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/tmp/am",
- "slider.cluster.directory.permissions": "0770",
- "slider.data.directory.permissions": "0770"
- },
-
- "options": {
- "slider.am.monitoring.enabled": "false",
- "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz",
- "slider.container.failure.threshold": "5",
- "slider.container.failure.shortlife": "60",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_slider_slider_test_cluster_lifecycle",
- "zookeeper.hosts": "sandbox",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://sandbox:8020",
- "site.fs.default.name": "hdfs://sandbox:8020",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- },
-
- "diagnostics": {
- "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
- "create.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by slider",
- "create.hadoop.build.info": "2.3.0",
- "create.time.millis": "1393512091276",
- "create.time": "27 Feb 2014 14:41:31 GMT"
- },
-
- "info": {
- "slider.am.restart.supported": "false",
- "live.time": "27 Feb 2014 14:41:56 GMT",
- "live.time.millis": "1393512116881",
- "status.time": "27 Feb 2014 14:42:08 GMT",
- "status.time.millis": "1393512128726",
- "yarn.vcores": "32",
- "yarn.memory": "2048",
- "status.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by slider",
- "status.hadoop.build.info": "2.3.0",
- "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
- },
-
- "statistics": {
-
- "cluster": {
- "containers.unknown.completed": 0,
- "containers.start.completed": 3,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.surplus": 0
-
- },
- "roles": {
- "worker": {
- "containers.start.completed": 0,
- "containers.live": 2,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 2,
- "containers.requested": 0
- },
- "master": {
- "containers.start.completed": 0,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 1,
- "containers.requested": 0
- }
- }
- },
-
- "instances": {
- "slider": [ "container_1393511571284_0002_01_000001" ],
- "master": [ "container_1393511571284_0002_01_000003" ],
- "worker": [
- "container_1393511571284_0002_01_000002",
- "container_1393511571284_0002_01_000004"
- ]
- },
-
- "roles": {
- "worker": {
- "yarn.memory": "768",
- "role.instances": "0",
- "role.name": "worker",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1"
- },
- "slider": {
- "yarn.memory": "256",
- "role.instances": "1",
- "role.name": "slider",
- "jvm.heapsize": "256M",
- "yarn.vcores": "1"
- },
- "master": {
- "yarn.memory": "1024",
- "role.instances": "0",
- "role.name": "master",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1"
- }
- },
-
-
- "clientProperties": {
- "fs.defaultFS": "hdfs://sandbox:8020",
- "hbase.cluster.distributed": "true",
- "hbase.master.info.port": "0",
- "hbase.master.port": "0",
- "hbase.master.startup.retainassign": "true",
- "hbase.regionserver.hlog.tolerable.lowreplication": "1",
- "hbase.regionserver.info.port": "0",
- "hbase.regionserver.port": "0",
- "hbase.rootdir": "hdfs://sandbox:8020/user/slider/.slider/cluster/test_cluster_lifecycle/database",
- "hbase.tmp.dir": "./hbase-tmp",
- "hbase.zookeeper.property.clientPort": "2181",
- "hbase.zookeeper.quorum": "sandbox",
- "zookeeper.znode.parent": "/yarnapps_slider_slider_test_cluster_lifecycle"
- },
-
-
- "clientfiles": {
- "hbase-site.xml": "site information for HBase",
- "log4.properties": "log4.property file"
- },
-
- "provider":{
- "load":0.4,
- "urls": {
- "master": ["http://node4:28209"],
- "worker": ["http://node4:28717", "http://node6:31268"]
- }
- },
-
- "status": {
- "live": {
- "worker": {
- "container_1394032374441_0001_01_000003": {
- "name": "container_1394032374441_0001_01_000003",
- "role": "worker",
- "roleId": 1,
- "createTime": 1394032384451,
- "startTime": 1394032384503,
- "released": false,
- "host": "192.168.1.88",
- "state": 3,
- "exitCode": 0,
- "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
- "diagnostics": "",
- "environment": [
- "HADOOP_USER_NAME=\"slider\"",
- "HBASE_LOG_DIR=\"/tmp/slider-slider\"",
- "HBASE_HEAPSIZE=\"256\"",
- "MALLOC_ARENA_MAX=\"4\"",
- "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
- ]
- },
- "container_1394032374441_0001_01_000002": {
- "name": "container_1394032374441_0001_01_000002",
- "role": "worker",
- "roleId": 1,
- "createTime": 1394032384451,
- "startTime": 1394032384552,
- "released": false,
- "host": "192.168.1.86",
- "state": 3,
- "exitCode": 0,
- "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
- "diagnostics": "",
- "environment": [
- "HADOOP_USER_NAME=\"slider\"",
- "HBASE_LOG_DIR=\"/tmp/slider-slider\"",
- "HBASE_HEAPSIZE=\"256\"",
- "MALLOC_ARENA_MAX=\"4\"",
- "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
- ]
- }
- },
- "slider": {
- "container_1394032374441_0001_01_000001": {
- "name": "container_1394032374441_0001_01_000001",
- "role": "slider",
- "roleId": 0,
- "createTime": 0,
- "startTime": 0,
- "released": false,
- "host": "slider-8.local",
- "state": 3,
- "exitCode": 0,
- "command": "",
- "diagnostics": ""
- }
- },
- "master": {
- "container_1394032374441_0001_01_000004": {
- "name": "container_1394032374441_0001_01_000004",
- "role": "master",
- "roleId": 2,
- "createTime": 1394032384451,
- "startTime": 1394032384573,
- "released": false,
- "host": "192.168.1.86",
- "state": 3,
- "exitCode": 0,
- "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR master start 1><LOG_DIR>/master.txt 2>&1 ; ",
- "diagnostics": "",
- "environment": [
- "HADOOP_USER_NAME=\"slider\"",
- "HBASE_LOG_DIR=\"/tmp/slider-slider\"",
- "HBASE_HEAPSIZE=\"256\"",
- "MALLOC_ARENA_MAX=\"4\"",
- "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
- ]
- }
- }
- },
- "failed": {
-
- },
-
- "rolestatus": {
- "worker": {
- "role.instances": "2",
- "role.requested.instances": "0",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "2",
- "role.releasing.instances": "0",
- "role.failed.instances": "1"
- },
- "slider": {
- "role.instances": "1",
- "role.requested.instances": "0",
- "role.name": "slider",
- "role.actual.instances": "1",
- "role.releasing.instances": "0",
- "role.failed.instances": "0"
- },
- "master": {
- "role.instances": "1",
- "role.requested.instances": "1",
- "role.name": "master",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "0",
- "role.releasing.instances": "0",
- "role.failed.instances": "0"
- }
- }
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/redesign.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/redesign.md b/src/site/markdown/configuration/redesign.md
deleted file mode 100644
index 0006662..0000000
--- a/src/site/markdown/configuration/redesign.md
+++ /dev/null
@@ -1,478 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Cluster Specification
-
-### Notation:
-
-In this document, a full path to a value is represented as a path
-`options/zookeeper.port` ; an assigment as `options/zookeeper.port=2181`.
-
-A wildcard indicates all entries matching a path: `options/zookeeper.*`
-or `/roles/*/yarn.memory`
-
-
-## History
-
-The Slider cluster specification was implicitly defined in the file
-`org.apache.slider.api.ClusterDescription`. It had a number of roles
-
-1. Persistent representaton of cluster state
-1. Internal model of desired cluster state within the Application Master.
-1. Dynamic representation of current cluster state when the AM
-was queried, marshalled over the network as JSON.
-1. Description of updated state when reconfiguring a running cluster.
-
-Initially the dynamic status included a complete history of all containers
--this soon highlit some restrictions on the maximum size of a JSON-formatted
-string in Hadoop's "classic" RPC: 32K, after which the string was silently
-truncated. Accordingly, this history was dropped.
-
-Having moved to Protocol Buffers as the IPC wire format, with a web view
-alongside, this history could be reconsidered.
-
-The initial design place most values into the root entry, and relied
-on Jaxon introspection to set and retrieve the values -it was a
-Java-first specification, with no external specificatin or regression tests.
-
-As the number of entries in the root increased, the design switched to storing
-more attributes into specific sections *under* the root path:
-
-* `info`: read-only information about the cluster.
-* `statistics`: Numeric statistics about the cluster
-
-# Sections
-
-## Root
-
-Contains various string and integer values
-
- "version": "1.0",
- "name": "test_cluster_lifecycle",
- "type": "hbase",
- "state": 3,
- "createTime": 1393512091276,
- "updateTime": 1393512117286,
- "originConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot",
- "generatedConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/generated",
- "dataPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database",
-
-
-* `version`: version of the JSON file. Not currently used
-to validate version compatibility; at this point in time
-releases may not be able to read existing .json files.
-
-* `name`: cluster name
-* `type`: reference to the provider type -this triggers a Hadoop configuration
-property lookup to find the implementation classes.
-* `state`: an enumeration value of the cluster state.
-
- int STATE_INCOMPLETE = 0;
- int STATE_SUBMITTED = 1;
- int STATE_CREATED = 2;
- int STATE_LIVE = 3;
- int STATE_STOPPED = 4;
- int STATE_DESTROYED = 5;
-
- Only two states are persisted, "incomplete" and "created", though more
- are used internally.
- The `incomplete` state is used during cluster create/build,
- allowing an incomplete JSON file to be written
- -so minimising the window for race conditions on cluster construction.
-
-* `createTime` and `updateTime`: timestamps, informative only.
- The `createTime` value is duplicated in `/info/createTimeMillis`
-* `originConfigurationPath`, `generatedConfigurationPath`, `dataPath` paths
-used internally -if changed the cluster may not start.
-
-*Proposed*:
-1. Move all state bar `name` and cluster state
-into a section `/slider-internal`.
-1. The cluster state is moved from an enum to a simple
- boolean, `valid`, set to true when the cluster JSON
- has been fully constructed.
-
-## `/info`
-
-Read-only list of information about the application. Generally this is
-intended to be used for debugging and testing.
-
-### Persisted values: static information about the file history
-
- "info" : {
- "create.hadoop.deployed.info" : "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
- "create.application.build.info" : "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
- "create.hadoop.build.info" : "2.3.0",
- "create.time.millis" : "1393512091276",
- },
-
-*Proposed*: move persisted info K-V pairs to a section `/diagnostics`.
-
-### Dynamic values:
-
-
- whether the AM supports service restart without killing all the containers hosting
- the role instances:
-
- "slider.am.restart.supported" : "false",
-
- timestamps of the cluster going live, and when the status query was made
-
- "live.time" : "27 Feb 2014 14:41:56 GMT",
- "live.time.millis" : "1393512116881",
- "status.time" : "27 Feb 2014 14:42:08 GMT",
- "status.time.millis" : "1393512128726",
-
- yarn data provided to the AM
-
- "yarn.vcores" : "32",
- "yarn.memory" : "2048",
-
- information about the application and hadoop versions in use. Here
- the application was built using Hadoop 2.3.0, but is running against the version
- of Hadoop built for HDP-2.
-
- "status.application.build.info" : "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
- "status.hadoop.build.info" : "2.3.0",
- "status.hadoop.deployed.info" : "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
-
-
- ## `instances`
-
- Information about the live containers in a cluster
-
- "instances": {
- "slider": [ "container_1393511571284_0002_01_000001" ],
- "master": [ "container_1393511571284_0002_01_000003" ],
- "worker": [
- "container_1393511571284_0002_01_000002",
- "container_1393511571284_0002_01_000004"
- ]
- },
-
-There's no information about location, nor is there any history about containers
-that are no longer part of the cluster (i.e. failed & released containers).
-
-It could be possible to include a list of previous containers,
-though Slider would need to be selective about how many to store
-(or how much detail to retain) on those previous containers.
-
-Perhaps the list could be allowed to grow without limit, but detail
-only preserved on the last 100. If more containers fail than that,
-there is likely to be a problem which the most recent containers
-will also display.
-
-*Proposed*
-
-1. Return to the full serialization of container state -but only for running containers.
-1. Have a list of failed containers, but only include last 8; make it a rolling
-buffer. This avoids a significantly failing role to overload the status document.
-
-
- ## `statistics`
-
- Statistics on each role.
-
- They can be divided into counters that only increase
-
- "containers.start.completed": 0,
- "containers.start.failed": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.requested": 0
-
-and those that vary depending upon the current state
-
- "containers.live": 0,
- "containers.active.requests": 0,
- "containers.desired": 0,
-
-
-* Propose: move these values out of statistics into some other section, as they
-are state, not statistics*
-
-
- "statistics": {
- "worker": {
- "containers.start.completed": 0,
- "containers.live": 2,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 2,
- "containers.requested": 0
- },
- "slider": {
- "containers.unknown.completed": 0,
- "containers.start.completed": 3,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.surplus": 0
- },
- "master": {
- "containers.start.completed": 0,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 1,
- "containers.requested": 0
- }
- },
-
-The `/statistics/slider` section is unusual in that it provides the aggregate statistics
-of the cluster -this is not obvious. A different name could be used -but
-again, there's a risk of clash with or confusion with a role.
-
-Better to have a specific `/statistics/cluster` element,
-and to move the roles' statistics under `/statistics/roles`:
-
- "statistics": {
- "cluster": {
- "containers.unknown.completed": 0,
- "containers.start.completed": 3,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.surplus": 0
-
- },
- "roles": {
- "worker": {
- "containers.start.completed": 0,
- "containers.live": 2,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 2,
- "containers.requested": 0
- },
- "master": {
- "containers.start.completed": 0,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 1,
- "containers.requested": 0
- }
- }
- },
-
-This approach allows extra statistics sections to be added (perhaps
-by providers), without any changes to the toplevel section.
-
-## Options
-
-A list of options used by Slider and its providers to build up the AM
-and the configurations of the deployed service components
-
-
- "options": {
- "zookeeper.port": "2181",
- "site.hbase.master.startup.retainassign": "true",
- "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz",
- "site.fs.defaultFS": "hdfs://sandbox:8020",
- "slider.container.failure.threshold": "5",
- "site.fs.default.name": "hdfs://sandbox:8020",
- "slider.cluster.directory.permissions": "0770",
- "slider.am.monitoring.enabled": "false",
- "zookeeper.path": "/yarnapps_slider_stevel_test_cluster_lifecycle",
- "slider.tmp.dir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/tmp/am",
- "slider.data.directory.permissions": "0770",
- "zookeeper.hosts": "sandbox",
- "slider.container.failure.shortlife": "60"
- },
-
-Some for these options have been created by slider itself ("slider.tmp.dir")
-for internal use -and are cluster specific. If/when the ability to use
-an existing json file as a template for a new cluster is added, having these
-options in the configuration will create problems
-
-
-# Proposed Changes
-
-
-## Move Slider internal state to `/slider-internal`
-
-Move all slider "private" data to an internal section,`/slider-internal`
-including those in the toplevel directory and in `/options`
-
-## Allow `/options` and `roles/*/` options entries to take the value "null".
-
-This would be a definition that the value must be defined before the cluster
-can start. Provider templates could declare this.
-
-## Make client configuration retrieval hierarchical -and maybe move out of the
-status
-
-The current design assumes that it is a -site.xml file being served up. This
-does not work for alternate file formats generated by the Provider.
-
-## Role Options
-
-The `/roles/$ROLENAME/` clauses each provide options for a
-specific role.
-
-This includes
-1. `role.instances`: defines the number of instances of a role to create
-1. `env.` environment variables for launching the container
-1. `yarn.` properties to configure YARN requests.
-1. `jvm.heapsize`: an option supported by some providers to
-fix the heap size of a component.
-1. `app.infoport`: an option supported by some providers (e.g. HBase)
-to fix the port to which a role (master or worker) binds its web UI.
-
-
-
- "worker": {
- "yarn.memory": "768",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "0",
- "role.name": "worker",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1",
- "app.infoport": "0"
- },
-
-In a live cluster, the role information also includes status information
-about the cluster.
-
- "master": {
- "yarn.memory": "1024",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "0",
- "role.requested.instances": "0",
- "role.name": "master",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "0",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1",
- "role.releasing.instances": "0",
- "role.failed.instances": "0",
- "app.infoport": "0"
- }
-
-The role `slider` represents the Slider Application Master itself.
-
-
- "slider": {
- "yarn.memory": "256",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "1",
- "role.name": "slider",
- "jvm.heapsize": "256M",
- "yarn.vcores": "1",
- },
-
-### Proposed:
-1. move all dynamic role status to its own clauses.
-1. use a simple inheritance model from `/options`
-1. don't allow role entries to alter the cluster state.
-
-### Proposed: `/clientProperties` continues return Key-val pairs
-
-The `/clientProperties` section will remain, with key-val pairs of type
-string, the expectation being this is where providers can insert specific
-single attributes for client applications.
-
-These values can be converted to application-specific files on the client,
-in code -as done today in the Slider CLI-, or via template expansion (beyond
-the scope of this document.
-
-
-
-### Proposed: alongside `/clientProperties` comes `/clientfiles`
-
-This section will list all files that an application instance can generate
-for clients, along with with a description.
-
- "/clientfiles/hbase-site.xml": "site information for HBase"
- "/clientfiles/log4.properties": "log4.property file"
-
-A new CLI command would be added to retrieve a client file.
-1. The specific file must be named.
-1. If it is not present, an error must be raised.
-1. If it is present, it is downloaded and output to the console/to a named
-destination file/directory `--outfile <file>` and `--outdir <dir>`
-1. If the `--list` argument is provided, the list of available files is
-returned (e.g.)
-
- hbase-site.xml: site information for HBase
- log4.properties: log4.property file
-
-*No attempt to parse/process the body of the messages will be returned.*
-
-In a REST implementation of the client API, /clientconf would be a path
-to the list of options; each file a path underneath.
-
-Client configuration file retrieval outside the status completely;
-the status just lists the possible values; a separate call returns them.
-
-This will permit binary content to be retrieved, and avoid any marshalling
-problems and inefficiencies.
-
-With this change, there will now be two ways to generate client configuration
-files
-
-* Client-side: as today
-* Server-side: via the provider
-
-Client side is more extensible as it allows for arbitrary clients; server-side
-is restricted to those files which the application provider is capable of
-generating. The advantage of the server-side option is that for those files
-about which the provider is aware of, they will be visible through the
-REST and Web UIs, so trivially retrieved.
-
-### Stop intermixing role specification with role current state
-
-Create a new section, `rolestatus`, which lists the current status
-of the roles: how many are running vs requested, how many are being
-released.
-
-There's some overlap here with the `/statistics` field, so we should
-either merge them or clearly separate the two. Only the `role.failed`
-properties match entries in the statistics -perhaps they should be cut.
-
-#### provider-specific status
-
-Allow providers to publish information to the status, in their
-own section.
-
-There already is support for providers updating the cluster status
-in Slider 12.1 and earlier, but it has flaws
-
-A key one is that it is done sychronously on a `getStatus()` call;
-as providers may perform a live query of their status (example, the HBase
-provider looks up the Web UI ports published by HBase to zookeeper),
-there's overhead, and if the operation blocks (example: when HBase hasn't
-ever been deployed and the zookeeper path is empty), then the status
-call blocks.
-
-*Proposed:*
-
-1. There is a specific `/provider` section
-1. There's no restriction on what JSON is permitted in this section.
-1. Providers may make their own updates to the application state to read and
-write this block -operations that are asynchronous to any status queries.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/resolved-resources.json
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/resolved-resources.json b/src/site/markdown/configuration/resolved-resources.json
deleted file mode 100644
index 5299897..0000000
--- a/src/site/markdown/configuration/resolved-resources.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
-
- "metadata": {
- "description": "example of a resources file"
- },
-
- "global": {
- "yarn.vcores": "1",
- "yarn.memory": "512"
- },
-
- "components": {
- "master": {
- "instances": "1",
- "yarn.memory": "1024"
- },
- "worker": {
- "instances":"5"
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/configuration/specification.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/configuration/specification.md b/src/site/markdown/configuration/specification.md
deleted file mode 100644
index 2ee5c50..0000000
--- a/src/site/markdown/configuration/specification.md
+++ /dev/null
@@ -1,512 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Specification of the "Cluster Description"
-
-* This is partially obsolete. Slider still returns the Slider Cluster Description
-as changing it will break most of the unit tests -once these are updated
-this document will be completely obsolete and replaced with a new one.
-
-
-### Notation:
-
-In this document, a full path to a value is represented as a path
-`options/zookeeper.port` ; an assigment as `options/zookeeper.port=2181`.
-
-A wildcard indicates all entries matching a path: `options/zookeeper.*`
-or `/roles/*/yarn.memory`
-
-
-## Core Concepts
-
-The specificaton of an application instance is defined in an application instance
-directory, `${user.home}/.slidera/clusters/${clustername}/cluster.json`)
-
-
-## Sections for specifying and describing cluster state
-
-The cluster desciption is hierarchal, with standardized sections.
-
-Different sections have one of three roles.
-
-1. Storage and specification of internal properties used to define a cluster -properties
-that should not be modified by users -doing so is likely to render the
-cluster undeployable.
-
-1. Storage and specification of the components deployed by Slider.
-These sections define options for the deployed application, the size of
-the deployed application, attributes of the deployed roles, and customizable
-aspects of the Slider application master.
-
- This information defines the *desired state* of a cluster.
-
- Users may edit these sections, either via the CLI, or by directly editing the `cluster.json` file of
- a frozen cluster.
-
-1. Status information provided by a running cluster. These include:
- information about the cluster, statistics, information about reach role in
- the cluster -as well as other aspects of the deployment.
-
- This information describes the *actual state* of a cluster.
-
-Using a common format for both the specification and description of a cluster
-may be confusing, but it is designed to unify the logic needed to parse
-and process cluster descriptions. There is only one JSON file to parse
--merely different sections of relevance at different times.
-
-## Role-by-role subsections
-
-A slider-deployed application consists of the single Slider application master,
-and one or more roles -specific components in the actual application.
-
-The `/roles` section contains a listing for each role,
-declaring the number of instances of each role desired,
-possibly along with some details defining the actual execution of the application.
-
-The `/statistics/roles/` section returns statistics on each role,
-while `/instances` has a per-role entry listing the YARN
-containers hosting instances.
-
-
-## Cluster information for applications
-
-The AM/application provider may generate information for use by client applications.
-
-There are three ways to provide this
-
-1. A section in which simple key-value pairs are provided for interpretation
-by client applications -usually to generate configuration documents
-2. A listing of files that may be provided directly to a client. The API to provide these files is not covered by this document.
-3. A provider-specific section in which arbitrary values and structures may be defined. This allows greater flexibility in the information that a provider can publish -though it does imply custom code to process this data on the client.
-
-
-# Persistent Specification Sections
-
-## "/" : root
-
-The root contains a limited number of key-value pairs,
-
-* `version`: string; required.
-The version of the JSON file, as an `x.y.z` version string.
- 1. Applications MUST verify that they can process a specific version.
- 1. The version number SHOULD be incremented in the minor "z" value
- after enhancements that are considered backwards compatible.
- Incompatible updates MUST be updated with a new "y" value.
- The final, "x" number, is to be reserved for major reworkings
- of the cluster specification itself (this document or its
- successors).
-
-* `name`: string; required. Cluster name;
-* `type`: string; required.
-Reference to the provider type -this triggers a Hadoop configuration
-property lookup to find the implementation classes.
-* `valid`: boolean; required.
-Flag to indicate whether or not a specification is considered valid.
-If false, the rest of the document is in an unknown state.
-
-## `/slider-internal`: internal confiugration
-
-Stores internal configuration options. These parameters
-are not defined in this document.
-
-## `/diagnostics`: diagnostics sections
-
-Persisted list of information about Slider.
-
-Static information about the file history
-
- "diagnostics" : {
- "create.hadoop.deployed.info" :
- "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
- "create.application.build.info" :
- "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
- "create.hadoop.build.info" : "2.3.0",
- "create.time.millis" : "1393512091276",
- },
-
-This information is not intended to provide anything other
-than diagnostics to an application; the values and their meaning
-are not defined. All applications MUST be able to process
-an empty or absent `/diagnostics` section.
-
-## Options: cluster options
-
-A persisted list of options used by Slider and its providers to build up the AM
-and the configurations of the deployed service components
-
-
- "options": {
- "slider.am.monitoring.enabled": "false",
- "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz",
- "slider.container.failure.threshold": "5",
- "slider.container.failure.shortlife": "60",
- "zookeeper.port": "2181",
- "zookeeper.path": "/yarnapps_slider_stevel_test_cluster_lifecycle",
- "zookeeper.hosts": "sandbox",
- "site.hbase.master.startup.retainassign": "true",
- "site.fs.defaultFS": "hdfs://sandbox:8020",
- "site.fs.default.name": "hdfs://sandbox:8020",
- "env.MALLOC_ARENA_MAX": "4",
- "site.hbase.master.info.port": "0",
- "site.hbase.regionserver.info.port": "0"
- },
-
-Many of the properties are automatically set by Slider when a cluster is constructed.
-They may be edited afterwards.
-
-
-### Standard Option types
-
-All option values MUST be strings.
-
-#### `slider.`
-All options that begin with `slider.` are intended for use by slider and
-providers to configure the Slider application master itself, and the
-application. For example, `slider.container.failure.threshold` defines
-the number of times a container must fail before the role (and hence the cluster)
-is considered to have failed. As another example, the zookeeper bindings
-such as `zookeeper.hosts` are read by the HBase and Ambari providers, and
-used to modify the applications' site configurations with application-specific
-properties.
-
-#### `site.`
-
-These are properties that are expected to be propagated to an application's
- `site` configuration -if such a configuration is created. For HBase, the
- site file is `hbase-site.xml`; for Accumulo it is `accumulo-site.xml`
-
-1. The destination property is taken by removing the prefix `site.`, and
-setting the shortened key with the defined value.
-1. Not all applications have the notion of a site file; These applications MAY
-ignore the settings.
-1. Providers MAY validate site settings to recognise invalid values. This
-aids identifying and diagnosing startup problems.
-
-#### `env.`
-
-These are options to configure environment variables in the roles. When
-a container is started, all `env.` options have the prefix removed, and
-are then set as environment variables in the target context.
-
-1. The Slider AM uses these values to configure itself, after following the
-option/role merge process.
-1. Application providers SHOULD follow the same process.
-
-
-## '/roles': role declarations
-
-The `/roles/$ROLENAME/` clauses each provide options for a
-specific role.
-
-This includes
-1. `role.instances`: defines the number of instances of a role to create
-1. `env.` environment variables for launching the container
-1. `yarn.` properties to configure YARN requests.
-1. `jvm.heapsize`: an option supported by some providers to
-fix the heap size of a component.
-
-
- "worker": {
- "yarn.memory": "768",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "0",
- "role.name": "worker",
- "role.failed.starting.instances": "0",
- "jvm.heapsize": "512M",
- "yarn.vcores": "1",
- },
-
-
-The role `slider` represents the Slider Application Master itself.
-
-
- "slider": {
- "yarn.memory": "256",
- "env.MALLOC_ARENA_MAX": "4",
- "role.instances": "1",
- "role.name": "slider",
- "jvm.heapsize": "256M",
- "yarn.vcores": "1",
- },
-
-Providers may support a fixed number of roles -or they may support a dynamic
-number of roles defined at run-time, potentially from other data sources.
-
-## How `/options` and role options are merged.
-
-The options declared for a specific role are merged with the cluster-wide options
-to define the final options for a role. This is implemented in a simple
-override model: role-specific options can override any site-wide options.
-
-1. The options defined in `/options` are used to create the initial option
-map for each role.
-1. The role's options are then applied to the map -this may overwrite definitions
-from the `/options` section.
-1. There is no way to "undefine" a cluster option, merely overwrite it.
-1. The merged map is then used by the provider to create the component.
-1. The special `slider` role is used in the CLI to define the attributes of the AM.
-
-Options set on a role do not affect any site-wide options: they
-are specific to the invidual role being created.
-
-As such, overwriting a `site.` option may have no effect -or it it may
-change the value of a site configuration document *in that specific role instance*.
-
-### Standard role options
-
-* `role.instances` : number; required.
- The number of instances of that role desired in the application.
-* `yarn.vcores` : number.
- The number of YARN "virtual cores" to request for each role instance.
- The larger the number, the more CPU allocation -and potentially the longer
- time to satisfy the request and so instantiate the node.
- If the value '"-1"` is used -for any role but `slider`-the maximum value
- available to the application is requested.
-* `yarn.memory` : number.
- The number in Megabytes of RAM to request for each role instance.
- The larger the number, the more memory allocation -and potentially the longer
- time to satisfy the request and so instantiate the node.
- If the value '"-1"` is used -for any role but `slider`-the maximum value
- available to the application is requested.
-
-* `env.` environment variables.
-String environment variables to use when setting up the container
-
-### Provider-specific role options
-
-* `jvm.heapsize` -the amount of memory for a provider to allocate for
- a processes JVM. Example "512M". This option MAY be implemented by a provider.
-
-
-
-
-
-# Dynamic Information Sections
-
-These are the parts of the document that provide dynamic run-time
-information about an application. They are provided by the
-Slider Application Master when a request for the cluster status is issued.
-
-## `/info`
-
-Dynamic set of string key-value pairs containing
-information about the running application -as provided by th
-
-The values in this section are not normatively defined.
-
-Here are some standard values
-
-* `slider.am.restart.supported"` whether the AM supports service restart without killing all the containers hosting
- the role instances:
-
- "slider.am.restart.supported" : "false",
-
-* timestamps of the cluster going live, and when the status query was made
-
- "live.time" : "27 Feb 2014 14:41:56 GMT",
- "live.time.millis" : "1393512116881",
- "status.time" : "27 Feb 2014 14:42:08 GMT",
- "status.time.millis" : "1393512128726",
-
-* yarn data provided to the AM
-
- "yarn.vcores" : "32",
- "yarn.memory" : "2048",
-
-* information about the application and hadoop versions in use. Here
- the application was built using Hadoop 2.3.0, but is running against the version
- of Hadoop built for HDP-2.
-
- "status.application.build.info" : "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
- "status.hadoop.build.info" : "2.3.0",
- "status.hadoop.deployed.info" : "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
-
-
-As with the `/diagnostics` section, this area is primarily intended
-for debugging.
-
- ## `/instances`: instance list
-
- Information about the live containers in a cluster
-
- "instances": {
- "slider": [ "container_1393511571284_0002_01_000001" ],
- "master": [ "container_1393511571284_0002_01_000003" ],
- "worker": [
- "container_1393511571284_0002_01_000002",
- "container_1393511571284_0002_01_000004"
- ]
- },
-
-
-## `/status`: detailed dynamic state
-
-This provides more detail on the application including live and failed instances
-
-### `/status/live`: live role instances by container
-
- "cluster": {
- "live": {
- "worker": {
- "container_1394032374441_0001_01_000003": {
- "name": "container_1394032374441_0001_01_000003",
- "role": "worker",
- "roleId": 1,
- "createTime": 1394032384451,
- "startTime": 1394032384503,
- "released": false,
- "host": "192.168.1.88",
- "state": 3,
- "exitCode": 0,
- "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
- "diagnostics": "",
- "environment": [
- "HADOOP_USER_NAME=\"slider\"",
- "HBASE_LOG_DIR=\"/tmp/slider-slider\"",
- "HBASE_HEAPSIZE=\"256\"",
- "MALLOC_ARENA_MAX=\"4\"",
- "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
- ]
- }
- }
- failed : {}
- }
-
-All live instances MUST be described in `/status/live`
-
-Failed clusters MAY be listed in the `/status/failed` section, specifically,
-a limited set of recently failed clusters SHOULD be provided.
-
-Future versions of this document may introduce more sections under `/status`.
-
-### `/status/rolestatus`: role status information
-
-This lists the current status of the roles:
-How many are running vs requested, how many are being
-released.
-
-
- "rolestatus": {
- "worker": {
- "role.instances": "2",
- "role.requested.instances": "0",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "2",
- "role.releasing.instances": "0",
- "role.failed.instances": "1"
- },
- "slider": {
- "role.instances": "1",
- "role.requested.instances": "0",
- "role.name": "slider",
- "role.actual.instances": "1",
- "role.releasing.instances": "0",
- "role.failed.instances": "0"
- },
- "master": {
- "role.instances": "1",
- "role.requested.instances": "1",
- "role.name": "master",
- "role.failed.starting.instances": "0",
- "role.actual.instances": "0",
- "role.releasing.instances": "0",
- "role.failed.instances": "0"
- }
- }
-
-
-### `/status/provider`: provider-specific information
-
-Providers MAY publish information to the `/status/provider` section.
-
-1. There's no restriction on what JSON is permitted in this section.
-1. Providers may make their own updates to the application state to read and
-write this block -operations that are asynchronous to any status queries.
-
-
-
-## `/statistics`: aggregate statistics
-
-Statistics on the cluster and each role in the cluster
-
-Better to have a specific `/statistics/cluster` element,
-and to move the roles' statistics under `/statistics/roles`:
-
- "statistics": {
- "cluster": {
- "containers.unknown.completed": 0,
- "containers.start.completed": 3,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.surplus": 0
- },
- "roles": {
- "worker": {
- "containers.start.completed": 0,
- "containers.live": 2,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 2,
- "containers.requested": 0
- },
- "master": {
- "containers.start.completed": 0,
- "containers.live": 1,
- "containers.start.failed": 0,
- "containers.active.requests": 0,
- "containers.failed": 0,
- "containers.completed": 0,
- "containers.desired": 1,
- "containers.requested": 0
- }
- }
- },
-
-`/statistics/cluster` provides aggregate statistics for the entire cluster.
-
-Under `/statistics/roles` MUST come an entry for each role in the cluster.
-
-All simple values in statistics section are integers.
-
-
-### `/clientProperties`
-
-The `/clientProperties` section contains key-val pairs of type
-string, the expectation being this is where providers can insert specific
-single attributes for client applications.
-
-These values can be converted to application-specific files on the client,
-in code -as done today in the Slider CLI-, or via template expansion (beyond
-the scope of this document.
-
-
-### `/clientfiles`
-
-This section list all files that an application instance MAY generate
-for clients, along with with a description.
-
- "/clientfiles/hbase-site.xml": "site information for HBase"
- "/clientfiles/log4.properties": "log4.property file"
-
-Client configuration file retrieval is by other means; this
-status operation merely lists files that are available;
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/debugging.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/debugging.md b/src/site/markdown/debugging.md
deleted file mode 100644
index fdc2d94..0000000
--- a/src/site/markdown/debugging.md
+++ /dev/null
@@ -1,92 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Debugging Apache Slider
-There are a number of options available to you for debugging Slider applications. They include:
-
-* Using Slider logging
-* IDE-based remote debugging of the Application Master
-
-## Using Slider logging
-There are a number of options for viewing the generated log files:
-
-1. Using a web browser
-2. Accessing the host machine
-
-### Using a web browser
-
-The log files are accessible via the Yarn Resource Manager UI. From the main page (e.g. `http://${YARN_RESOURCE_MGR_HOST}:8088`),
-click on the link for the application instance of interest, and then click on the `logs` link.
-This will present you with a page with links to the `slider-err.txt` file and the `slider-out.txt` file.
-The former is the file you should select -it is where the applicaton logs go
-Once the log page is presented, click on the link at the top of the page ("Click here for full log") to view the entire file.
-
-If the file `slider-out.txt` is empty, then examine `slider-err.txt` -an empty
-output log usually means that the java process failed to start -this should be
-logged in the error file.
-
-
-### Accessing the host machine
-
-If access to other log files is required, there is the option of logging in
- to the host machine on which the application component is running
- -provided you have the correct permissions.
-
-The root directory for all YARN associated files is the value of `yarn.nodemanager.log-dirs` in `yarn-site.xml` - e.g. `/hadoop/yarn/log`.
-Below the root directory you will find an application and container sub-directory (e.g. `/application_1398372047522_0009/container_1398372047522_0009_01_000001/`).
-Below the container directory you will find any log files associated with the processes running in the given Yarn container.
-
-Within a container log the following files are useful while debugging the application.
-
-**agent.log**
-
-E.g. `application_1398098639743_0024/container_1398098639743_0024_01_000003/infra/log/agent.log`
-This file contains the logs from the Slider-Agent.
-
-**application component log**
-
-E.g. `./log/application_1398098639743_0024/container_1398098639743_0024_01_000003/app/log/hbase-yarn-regionserver-c6403.ambari.apache.org.log`
-
-The location of the application log is defined by the application. "${AGENT_LOG_ROOT}" is a symbol available to the app developers to use as a root folder for logging.
-
-**agent operations log**
-
-E.g. ./log/application_1398098639743_0024/container_1398098639743_0024_01_000003/app/command-log/
-
-The command logs produced by the slider-agent are available in the `command-log` folder relative to `${AGENT_LOG_ROOT}/app`
-
-Note that the *fish* shell is convenient for debugging, as `cat log/**/slider-out.txt` will find the relevant output file
-irrespective of what the path leading to it is.
-
-## IDE-based remote debugging of the Application Master
-
-For situations in which the logging does not yield enough information to debug an issue,
-the user has the option of specifying JVM command line options for the
-Application Master that enable attaching to the running process with a debugger
-(e.g. the remote debugging facilities in Eclipse or Intellij IDEA).
-In order to specify the JVM options, edit the application configuration file
-(the file specified as the `--template` argument value on the command line for cluster creation)
-and specify the `jvm.opts` property for the `slider-appmaster` component:
-
- `"components": {
- "slider-appmaster": {
- "jvm.heapsize": "256M",
- "jvm.opts": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
- },
- ...`
-
-You may specify `suspend=y` in the line above if you wish to have the application master process wait for the debugger to attach before beginning its processing.
[39/50] [abbrv] git commit: SLIDER-173 When host is specified in
ContainerRequest, relaxLocality should be false
Posted by st...@apache.org.
SLIDER-173 When host is specified in ContainerRequest, relaxLocality should be false
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/22c9689e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/22c9689e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/22c9689e
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 22c9689e648a13f09dda9e78337ebd5afe7f0a47
Parents: 9619c33
Author: tedyu <yu...@gmail.com>
Authored: Wed Jun 25 19:13:07 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Jun 25 19:13:07 2014 -0700
----------------------------------------------------------------------
.../slider/server/appmaster/state/OutstandingRequest.java | 2 +-
.../org/apache/slider/server/appmaster/state/RoleHistory.java | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22c9689e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 7d3e427..45dd9d1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -108,7 +108,7 @@ public final class OutstandingRequest {
if (node != null) {
hosts = new String[1];
hosts[0] = node.hostname;
- relaxLocality = true;
+ relaxLocality = false;
locationSpecified = true;
// tell the node it is in play
node.getOrCreate(roleId);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/22c9689e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 68e7693..990d735 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,6 +29,7 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.RoleHistoryWriter;
+import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -468,12 +469,17 @@ public class RoleHistory {
NodeInstance nodeInstance = null;
List<NodeInstance> targets = getNodesForRoleId(roleKey);
+ int cnt = targets == null ? 0 : targets.size();
+ Log.info("There're " + cnt + " nodes to consider for " + role.getName());
while (targets != null && !targets.isEmpty() && nodeInstance == null) {
NodeInstance head = targets.remove(0);
if (head.getActiveRoleInstances(roleKey) == 0) {
nodeInstance = head;
}
}
+ if (nodeInstance == null) {
+ Log.debug("No node selected for " + role.getName());
+ }
return nodeInstance;
}
[33/50] [abbrv] git commit: Merge branch
'feature/SLIDER-160_Improve_slider_functional_testing' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-160_Improve_slider_functional_testing' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/18667336
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/18667336
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/18667336
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 186673363c32a644d5ebaa21e3e2f7e3150115d6
Parents: 3be6c86 bab8867
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 18:10:28 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 18:10:28 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 68 +++++++++++++
.../funtest/framework/CommandTestBase.groovy | 9 +-
.../funtest/framework/FileUploader.groovy | 100 +++++++++++++++++++
.../funtest/framework/FuntestProperties.groovy | 12 ++-
.../slider/funtest/framework/SudoClosure.groovy | 54 ++++++++++
.../lifecycle/AgentCommandTestBase.groovy | 39 +++-----
.../lifecycle/TestAppsThroughAgent.groovy | 5 +-
.../lifecycle/TestClusterBuildDestroy.groovy | 2 +-
src/test/clusters/c6401/slider/log4j.properties | 83 +++++++++++++++
.../clusters/c6401/slider/slider-client.xml | 71 +++++++++++++
10 files changed, 403 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
[03/50] [abbrv] git commit: SLIDER-126. Slider-Agent and Agent
Provider need to handle component instance install/start failure gracefully
Posted by st...@apache.org.
SLIDER-126. Slider-Agent and Agent Provider need to handle component instance install/start failure gracefully
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3aca57d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3aca57d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3aca57d2
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 3aca57d2c0d8e06fc720b023583d5a7ce1f4273e
Parents: 7003c06
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Jun 21 18:09:42 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Jun 21 18:09:42 2014 -0700
----------------------------------------------------------------------
slider-agent/conf/agent.ini | 2 +
.../src/main/python/agent/AgentConfig.py | 21 +++
slider-agent/src/main/python/agent/Constants.py | 3 +
.../src/main/python/agent/Controller.py | 23 ++-
slider-agent/src/main/python/agent/main.py | 4 +
.../src/test/python/agent/TestController.py | 30 ++++
slider-agent/src/test/python/agent/TestMain.py | 5 +-
.../providers/AbstractProviderService.java | 15 +-
.../slider/providers/ProviderService.java | 5 +-
.../slider/providers/agent/AgentKeys.java | 3 +
.../providers/agent/AgentLaunchParameter.java | 130 ++++++++++++++
.../providers/agent/AgentProviderService.java | 179 ++++++++++++++++---
.../slider/providers/agent/AgentRoles.java | 18 +-
.../apache/slider/providers/agent/Command.java | 13 +-
.../slider/providers/agent/CommandResult.java | 16 +-
.../providers/agent/ComponentInstanceState.java | 24 +++
.../slider/providers/agent/ContainerState.java | 41 +++++
.../providers/agent/HeartbeatMonitor.java | 116 ++++++++++++
.../server/appmaster/AMViewForProviders.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 32 +++-
.../slider/server/appmaster/state/AppState.java | 20 +++
.../test_command_log/appConfig_fast_no_reg.json | 29 +++
.../test_command_log/appConfig_no_hb.json | 29 +++
.../model/mock/MockProviderService.groovy | 4 +-
.../agent/TestAgentLaunchParameter.java | 76 ++++++++
.../providers/agent/TestHeartbeatMonitor.java | 136 ++++++++++++++
.../publisher/TestAgentProviderService.java | 5 +-
.../lifecycle/AgentCommandTestBase.groovy | 100 ++++++++---
.../lifecycle/TestAgentClusterLifecycle.groovy | 2 +-
.../funtest/lifecycle/TestAgentFailures.groovy | 103 +++++++++++
.../funtest/lifecycle/TestAgentFailures2.groovy | 103 +++++++++++
.../lifecycle/TestAppsThroughAgent.groovy | 90 ++++------
32 files changed, 1253 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/conf/agent.ini
----------------------------------------------------------------------
diff --git a/slider-agent/conf/agent.ini b/slider-agent/conf/agent.ini
index 87d73a7..b52bec9 100644
--- a/slider-agent/conf/agent.ini
+++ b/slider-agent/conf/agent.ini
@@ -25,6 +25,8 @@ heartbeat_path=/ws/v1/slider/agents/{name}/heartbeat
app_pkg_dir=app/definition
app_install_dir=app/install
app_run_dir=app/run
+app_dbg_cmd=
+debug_mode_enabled=true
app_task_dir=app/command-log
app_log_dir=app/log
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/src/main/python/agent/AgentConfig.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py
index e0981f6..91adfdd 100644
--- a/slider-agent/src/main/python/agent/AgentConfig.py
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -21,6 +21,9 @@ limitations under the License.
import ConfigParser
import StringIO
import os
+import logging
+
+logger = logging.getLogger()
config = ConfigParser.RawConfigParser()
content = """
@@ -37,6 +40,8 @@ heartbeat_path=/ws/v1/slider/agents/{name}/heartbeat
app_pkg_dir=app/definition
app_install_dir=app/install
app_run_dir=app/run
+app_dbg_cmd=
+debug_mode_enabled=true
app_task_dir=app/command-log
app_log_dir=app/log
@@ -79,6 +84,10 @@ class AgentConfig:
APP_INSTALL_DIR = "app_install_dir"
# the location to store component instance PID directories
APP_RUN_DIR = "app_run_dir"
+ # debug hint for agents
+ APP_DBG_CMD = "app_dbg_cmd"
+ # allow agent to operate in debug mode
+ DEBUG_MODE_ENABLED = "debug_mode_enabled"
# run time dir for command executions
APP_TASK_DIR = "app_task_dir"
@@ -137,6 +146,18 @@ class AgentConfig:
global config
return config.get(category, name)
+ def isDebugEnabled(self):
+ global config
+ enabled = config.get(AgentConfig.AGENT_SECTION, AgentConfig.DEBUG_MODE_ENABLED)
+ return enabled == "true";
+
+ def debugCommand(self):
+ global config
+ command = config.get(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD)
+ if command == None:
+ return ""
+ return command
+
def set(self, category, name, value):
global config
return config.set(category, name, value)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/src/main/python/agent/Constants.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index b937cd2..88cd564 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -27,3 +27,6 @@ ALLOCATED_PORTS = "allocated_ports"
FOLDERS = "folders"
AGENT_WORK_ROOT = "AGENT_WORK_ROOT"
AGENT_LOG_ROOT = "AGENT_LOG_ROOT"
+DO_NOT_REGISTER = "DO_NOT_REGISTER"
+DO_NOT_HEARTBEAT = "DO_NOT_HEARTBEAT"
+DO_NOT_HEARTBEAT_AFTER_ = "DO_NOT_HEARTBEAT_AFTER_"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index fe5760d..b5dca92 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -36,6 +36,7 @@ from ActionQueue import ActionQueue
from NetUtil import NetUtil
import ssl
import ProcessHelper
+import Constants
logger = logging.getLogger()
@@ -84,10 +85,29 @@ class Controller(threading.Thread):
logger.info("Server connection disconnected.")
pass
+ def processDebugCommandForRegister(self):
+ self.processDebugCommand(Constants.DO_NOT_REGISTER)
+ pass
+
+ def processDebugCommandForHeartbeat(self):
+ self.processDebugCommand(Constants.DO_NOT_HEARTBEAT)
+ pass
+
+ def processDebugCommand(self, command):
+ if self.config.isDebugEnabled() and self.config.debugCommand() == command:
+ ## Test support - sleep for 10 minutes
+ logger.info("Received debug command: "
+ + self.config.debugCommand() + " Sleeping for 10 minutes")
+ time.sleep(60*10)
+ pass
+ pass
+
def registerWithServer(self):
id = -1
ret = {}
+ self.processDebugCommandForRegister()
+
while not self.isRegistered:
try:
data = json.dumps(self.register.build(id))
@@ -170,7 +190,8 @@ class Controller(threading.Thread):
retry = False
certVerifFailed = False
- id = 0
+ self.processDebugCommandForHeartbeat()
+
while not self.DEBUG_STOP_HEARTBEATING:
if self.shouldStopAgent():
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index afe3595..3632157 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -176,6 +176,7 @@ def main():
parser.add_option("-l", "--label", dest="label", help="label of the agent", default=None)
parser.add_option("--host", dest="host", help="AppMaster host", default=None)
parser.add_option("--port", dest="port", help="AppMaster port", default=None)
+ parser.add_option("--debug", dest="debug", help="Agent debug hint", default="")
(options, args) = parser.parse_args()
if not 'AGENT_WORK_ROOT' in os.environ:
@@ -200,6 +201,9 @@ def main():
if options.port:
agentConfig.set(AgentConfig.SERVER_SECTION, "port", options.port)
+ if options.debug:
+ agentConfig.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, options.debug)
+
logFile = os.path.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
perform_prestart_checks(agentConfig)
ensure_folder_layout(agentConfig)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/src/test/python/agent/TestController.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index 8dc7458..8c671da 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -557,6 +557,36 @@ class TestController(unittest.TestCase):
#Conroller thread and the agent stop if the repeatRegistration flag is False
self.assertFalse(self.controller.repeatRegistration)
+ @patch("time.sleep")
+ def test_debugSetupForRegister(self, sleepMock):
+ original_value = self.controller.config
+ self.controller.config = AgentConfig("", "")
+ self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.DEBUG_MODE_ENABLED, "true")
+ self.controller.processDebugCommandForRegister()
+ self.controller.processDebugCommandForHeartbeat()
+ assert not sleepMock.called, 'sleep should not have been called'
+
+ self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_RERISTER")
+ self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_HEARTBEET")
+ self.controller.processDebugCommandForRegister()
+ self.controller.processDebugCommandForHeartbeat()
+ assert not sleepMock.called, 'sleep should not have been called'
+
+ self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_REGISTER")
+ self.controller.processDebugCommandForRegister()
+ assert sleepMock.called, 'sleep should have been called'
+
+ self.controller.processDebugCommandForHeartbeat()
+ assert sleepMock.call_count == 1, 'sleep should have been called once'
+
+ self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_HEARTBEAT")
+ self.controller.processDebugCommandForHeartbeat()
+ assert sleepMock.call_count == 2, 'sleep should have been called twice'
+
+ self.controller.config = original_value
+ pass
+
+
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
unittest.main()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index 5273623..179d1b4 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -259,11 +259,12 @@ class TestMain(unittest.TestCase):
self.assertTrue(start_mock.called)
class AgentOptions:
- def __init__(self, label, host, port, verbose):
+ def __init__(self, label, host, port, verbose, debug):
self.label = label
self.host = host
self.port = port
self.verbose = verbose
+ self.debug = debug
@patch.object(main, "setup_logging")
@patch.object(main, "bind_signal_handlers")
@@ -289,7 +290,7 @@ class TestMain(unittest.TestCase):
Controller_init_mock.return_value = None
isAlive_mock.return_value = False
parse_args_mock.return_value = (
- TestMain.AgentOptions("agent", "host1", "8080", True), [])
+ TestMain.AgentOptions("agent", "host1", "8080", True, ""), [])
tmpdir = tempfile.gettempdir()
#testing call without command-line arguments
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7c9b38e..bbad90f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -30,6 +30,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.RegisteredEndpoint;
import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
@@ -66,6 +67,7 @@ public abstract class AbstractProviderService
protected AgentRestOperations restOps;
protected RegistryViewForProviders registry;
protected ServiceInstanceData registryInstanceData;
+ protected AMViewForProviders amView;
protected URL amWebAPI;
public AbstractProviderService(String name) {
@@ -81,15 +83,20 @@ public abstract class AbstractProviderService
return amState;
}
+ public AMViewForProviders getAppMaster() {
+ return amView;
+ }
+
public void setAmState(StateAccessForProviders amState) {
this.amState = amState;
}
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg) {
+ RegistryViewForProviders reg, AMViewForProviders amView) {
this.amState = stateAccessor;
this.registry = reg;
+ this.amView = amView;
}
@Override
@@ -129,16 +136,16 @@ public abstract class AbstractProviderService
/**
* No-op implementation of this method.
- *
+ *
* {@inheritDoc}
*/
@Override
public void validateApplicationConfiguration(AggregateConf instance,
File confDir,
boolean secure) throws
- IOException,
+ IOException,
SliderException {
-
+
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index d77135c..1778197 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -31,6 +31,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
+import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
@@ -138,9 +139,11 @@ public interface ProviderService extends ProviderCore, Service,
* bind operation -invoked before the service is started
* @param stateAccessor interface offering read access to the state
* @param registry
+ * @param amView
*/
void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry);
+ RegistryViewForProviders registry,
+ AMViewForProviders amView);
/**
* Returns the agent rest operations interface.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 7136fd9..dea39ea 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -71,6 +71,7 @@ public interface AgentKeys {
String ARG_LABEL = "--label";
String ARG_HOST = "--host";
String ARG_PORT = "--port";
+ String ARG_DEBUG = "--debug";
String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/";
String AGENT_MAIN_SCRIPT = "agent/main.py";
@@ -88,6 +89,8 @@ public interface AgentKeys {
String COMPONENT_SCRIPT = "role.script";
String WAIT_HEARTBEAT = "wait.heartbeat";
String PYTHON_EXE = "python";
+ String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
+ String AGENT_INSTANCE_DEBUG_DATA = "agent.instance.debug.data";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
new file mode 100644
index 0000000..b839e58
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+class AgentLaunchParameter {
+ public static final Logger log =
+ LoggerFactory.getLogger(AgentLaunchParameter.class);
+ private static final String DEFAULT_PARAMETER = "";
+ private static final String ANY_COMPONENT = "ANY";
+ private static final String NONE_VALUE = "NONE";
+ private final Map<String, CommandTracker> launchParameterTracker;
+
+ public AgentLaunchParameter(String parameters) {
+ launchParameterTracker = parseExpectedLaunchParameters(parameters);
+ }
+
+ /**
+ * Get command for the component type
+ *
+ * @param componentName
+ *
+ * @return
+ */
+ public String getNextLaunchParameter(String componentName) {
+ if (launchParameterTracker != null) {
+ if (launchParameterTracker.containsKey(componentName)
+ || launchParameterTracker.containsKey(ANY_COMPONENT)) {
+ synchronized (this) {
+ CommandTracker indexTracker = null;
+ if (launchParameterTracker.containsKey(componentName)) {
+ indexTracker = launchParameterTracker.get(componentName);
+ } else {
+ indexTracker = launchParameterTracker.get(ANY_COMPONENT);
+ }
+
+ return indexTracker.getNextCommand();
+ }
+ }
+ }
+
+ return DEFAULT_PARAMETER;
+ }
+
+ /**
+ * Parse launch parameters of the form ANY:PARAM_FOR_FIRST:PARAM_FOR_SECOND:...:PARAM_FOR_REST|HBASE_MASTER:...
+ *
+ * E.g. ANY:DO_NOT_REGISTER:DO_NOT_HEARTBEAT:NONE For any container, first one gets DO_NOT_REGISTER second one gets
+ * DO_NOT_HEARTBEAT, then all of the rest get nothing
+ *
+ * E.g. HBASE_MASTER:FAIL_AFTER_START:NONE For HBASE_MASTER, first one gets FAIL_AFTER_START then "" for all
+ *
+ * @param launchParameters
+ *
+ * @return
+ */
+ Map<String, CommandTracker> parseExpectedLaunchParameters(String launchParameters) {
+ Map<String, CommandTracker> trackers = null;
+ if (launchParameters != null && launchParameters.length() > 0) {
+ String[] componentSpecificParameters = launchParameters.split(Pattern.quote("|"));
+ for (String componentSpecificParameter : componentSpecificParameters) {
+ if (componentSpecificParameter.length() != 0) {
+ String[] parameters = componentSpecificParameter.split(Pattern.quote(":"));
+
+ if (parameters.length > 1 && parameters[0].length() > 0) {
+
+ for (int index = 1; index < parameters.length; index++) {
+ if (parameters[index].equals(NONE_VALUE)) {
+ parameters[index] = DEFAULT_PARAMETER;
+ }
+ }
+
+ if (trackers == null) {
+ trackers = new HashMap<>(10);
+ }
+ String componentName = parameters[0];
+ CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));
+ trackers.put(componentName, tracker);
+ }
+ }
+ }
+ }
+
+ return trackers;
+ }
+
+ class CommandTracker {
+ private final int maxIndex;
+ private final String[] launchCommands;
+ private int currentIndex;
+
+ CommandTracker(String[] launchCommands) {
+ this.currentIndex = 0;
+ this.maxIndex = launchCommands.length - 1;
+ this.launchCommands = launchCommands;
+ }
+
+ String getNextCommand() {
+ String retVal = launchCommands[currentIndex];
+ if (currentIndex != maxIndex) {
+ currentIndex++;
+ }
+
+ return retVal;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 5ac142b..85945d8 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -91,7 +91,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
-/** This class implements the server-side aspects of an agent deployment */
+/** This class implements the server-side logic for application deployment
+ * through Slider application package
+ **/
public class AgentProviderService extends AbstractProviderService implements
ProviderCore,
AgentKeys,
@@ -106,13 +108,16 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final int MAX_LOG_ENTRIES = 20;
+ private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
private final Map<String, String> allocatedPorts = new ConcurrentHashMap<>();
+ private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
- private Map<String, ComponentInstanceState> componentStatuses = new HashMap<>();
+ private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metainfo = null;
private ComponentCommandOrder commandOrder = null;
+ private HeartbeatMonitor monitor;
private Map<String, String> workFolders =
Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
@@ -120,10 +125,15 @@ public class AgentProviderService extends AbstractProviderService implements
}
});
private Boolean canAnyMasterPublish = null;
+ private AgentLaunchParameter agentLaunchParameter = null;
+ /**
+ * Create an instance of AgentProviderService
+ */
public AgentProviderService() {
super("AgentProviderService");
setAgentRestOperations(this);
+ setHeartbeatMonitorInterval(DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
}
@Override
@@ -170,6 +180,9 @@ public class AgentProviderService extends AbstractProviderService implements
if (metainfo == null) {
synchronized (syncLock) {
if (metainfo == null) {
+ readAndSetHeartbeatMonitoringInterval(instanceDefinition);
+ initializeAgentDebugCommands(instanceDefinition);
+
metainfo = getApplicationMetainfo(fileSystem, appDef);
if (metainfo == null || metainfo.getServices() == null || metainfo.getServices().size() == 0) {
log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
@@ -177,6 +190,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
commandOrder = new ComponentCommandOrder(metainfo.getServices().get(0).getCommandOrder());
+ monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
+ monitor.start();
}
}
}
@@ -246,6 +261,12 @@ public class AgentProviderService extends AbstractProviderService implements
operation.add(ARG_PORT);
operation.add(getClusterInfoPropertyValue(StatusKeys.INFO_AM_WEB_PORT));
+ String debugCmd = agentLaunchParameter.getNextLaunchParameter(role);
+ if (debugCmd != null && debugCmd.length() != 0) {
+ operation.add(ARG_DEBUG);
+ operation.add(debugCmd);
+ }
+
launcher.addCommand(operation.build());
// initialize the component instance state
@@ -256,15 +277,66 @@ public class AgentProviderService extends AbstractProviderService implements
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
+ /**
+ * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ * @param instanceDefinition
+ */
+ private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
+ String hbMonitorInterval = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
+ Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
+ try {
+ setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
+ }catch (NumberFormatException e) {
+ log.warn(
+ "Bad value {} for {}. Defaulting to ",
+ hbMonitorInterval,
+ HEARTBEAT_MONITOR_INTERVAL,
+ DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
+ }
+ }
+
+ /**
+ * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ * @param instanceDefinition
+ */
+ private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
+ String launchParameterStr = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.AGENT_INSTANCE_DEBUG_DATA, "");
+ agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
+ }
+
+ @VisibleForTesting
protected Metainfo getMetainfo() {
return this.metainfo;
}
+ @VisibleForTesting
+ protected Map<String, ComponentInstanceState> getComponentStatuses() {
+ return componentStatuses;
+ }
+
+ @VisibleForTesting
protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String appDef) throws IOException {
return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
}
+ @VisibleForTesting
+ protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
+ this.heartbeatMonitorInterval = heartbeatMonitorInterval;
+ }
+
+ private int getHeartbeatMonitorInterval() {
+ return this.heartbeatMonitorInterval;
+ }
+
+ /**
+ * Publish a named config bag that may contain name-value pairs for app configurations such as hbase-site
+ * @param name
+ * @param description
+ * @param entries
+ */
protected void publishComponentConfiguration(String name, String description,
Iterable<Map.Entry<String, String>> entries) {
PublishedConfiguration pubconf = new PublishedConfiguration();
@@ -274,6 +346,10 @@ public class AgentProviderService extends AbstractProviderService implements
getAmState().getPublishedSliderConfigurations().put(name, pubconf);
}
+ /**
+ * Get a list of all hosts for all role/container per role
+ * @return
+ */
protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
amState.refreshClusterStatus();
return (Map<String, Map<String, ClusterNode>>)
@@ -293,6 +369,25 @@ public class AgentProviderService extends AbstractProviderService implements
}
/**
+ * Lost heartbeat from the container - release it and ask for a replacement
+ *
+ * @param label
+ *
+ * @return if release is requested successfully
+ */
+ protected boolean releaseContainer(String label) {
+ componentStatuses.remove(label);
+ try {
+ getAppMaster().refreshContainer(getContainerId(label), true);
+ } catch (SliderException e) {
+ log.info("Error while requesting container release for {}. Message: {}", label, e.getMessage());
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
* Run this service
*
* @param instanceDefinition component description
@@ -329,12 +424,18 @@ public class AgentProviderService extends AbstractProviderService implements
return true;
}
+ /**
+ * Handle registration calls from the agents
+ * @param registration
+ * @return
+ */
@Override
public RegistrationResponse handleRegistration(Register registration) {
RegistrationResponse response = new RegistrationResponse();
String label = registration.getHostname();
if (componentStatuses.containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
+ componentStatuses.get(label).setLastHeartbeat(System.currentTimeMillis());
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
@@ -342,31 +443,11 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
- private Command getCommand(String commandVal) {
- if (commandVal.equals(Command.START.toString())) {
- return Command.START;
- }
- if (commandVal.equals(Command.INSTALL.toString())) {
- return Command.INSTALL;
- }
-
- return Command.NOP;
- }
-
- private CommandResult getCommandResult(String commandResVal) {
- if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
- return CommandResult.COMPLETED;
- }
- if (commandResVal.equals(CommandResult.FAILED.toString())) {
- return CommandResult.FAILED;
- }
- if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
- return CommandResult.IN_PROGRESS;
- }
-
- throw new IllegalArgumentException("Unrecognized value " + commandResVal);
- }
-
+ /**
+ * Handle heartbeat response from agents
+ * @param heartBeat
+ * @return
+ */
@Override
public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
HeartBeatResponse response = new HeartBeatResponse();
@@ -391,6 +472,7 @@ public class AgentProviderService extends AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = componentStatuses.get(label);
+ componentStatus.setLastHeartbeat(System.currentTimeMillis());
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
if (isMaster &&
@@ -408,8 +490,8 @@ public class AgentProviderService extends AbstractProviderService implements
this.allocatedPorts.put(port.getKey(), port.getValue());
}
}
- CommandResult result = getCommandResult(report.getStatus());
- Command command = getCommand(report.getRoleCommand());
+ CommandResult result = CommandResult.getCommandResult(report.getStatus());
+ Command command = Command.getCommand(report.getRoleCommand());
componentStatus.applyCommandResult(result, command);
log.info("Component operation. Status: {}", result);
@@ -461,6 +543,12 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
+ /**
+ * Format the folder locations before publishing in the registry service
+ * @param folders
+ * @param containerId
+ * @param hostFqdn
+ */
private void processFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
for (String key : folders.keySet()) {
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
@@ -469,6 +557,11 @@ public class AgentProviderService extends AbstractProviderService implements
publishComponentConfiguration(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
}
+ /**
+ * Process return status for component instances
+ * @param heartBeat
+ * @param componentStatus
+ */
protected void processReturnedStatus(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
@@ -529,6 +622,11 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ /**
+ * Extract script path from the application metainfo
+ * @param roleName
+ * @return
+ */
protected String getScriptPathFromMetainfo(String roleName) {
String scriptPath = null;
List<Service> services = getMetainfo().getServices();
@@ -545,6 +643,11 @@ public class AgentProviderService extends AbstractProviderService implements
return scriptPath;
}
+ /**
+ * Is the role of type MASTER
+ * @param roleName
+ * @return
+ */
protected boolean isMaster(String roleName) {
List<Service> services = getMetainfo().getServices();
if (services.size() != 1) {
@@ -564,6 +667,11 @@ public class AgentProviderService extends AbstractProviderService implements
return false;
}
+ /**
+ * Can the role publish configuration
+ * @param roleName
+ * @return
+ */
protected boolean canPublishConfig(String roleName) {
List<Service> services = getMetainfo().getServices();
if (services.size() != 1) {
@@ -579,6 +687,10 @@ public class AgentProviderService extends AbstractProviderService implements
return false;
}
+ /**
+ * Can any master publish config explicitly, if not a random master is used
+ * @return
+ */
protected boolean canAnyMasterPublishConfig() {
if (canAnyMasterPublish == null) {
List<Service> services = getMetainfo().getServices();
@@ -609,6 +721,15 @@ public class AgentProviderService extends AbstractProviderService implements
return label.substring(0, label.indexOf(LABEL_MAKER));
}
+ /**
+ * Add install command to the heartbeat response
+ * @param roleName
+ * @param containerId
+ * @param response
+ * @param scriptPath
+ * @throws SliderException
+ */
+ @VisibleForTesting
protected void addInstallCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath)
throws SliderException {
assert getAmState().isApplicationLive();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
index d8aefc6..281895a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
@@ -18,7 +18,6 @@
package org.apache.slider.providers.agent;
-import org.apache.slider.common.SliderKeys;
import org.apache.slider.providers.ProviderRole;
import java.util.ArrayList;
@@ -27,22 +26,11 @@ import java.util.List;
public class AgentRoles {
/**
- * List of roles
+ * List of roles Agent provider does not have any roles by default. All roles are read from the application
+ * specification.
*/
protected static final List<ProviderRole> ROLES =
- new ArrayList<ProviderRole>();
-
- public static final int KEY_NODE =
- SliderKeys.ROLE_AM_PRIORITY_INDEX + 1;
- /**
- * Initialize role list
- */
-/*
- static {
- ROLES.add(new ProviderRole(AgentKeys.ROLE_NODE, KEY_NODE));
- }
-*/
-
+ new ArrayList<ProviderRole>();
public static List<ProviderRole> getRoles() {
return ROLES;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
index 541dcc2..cbeb69d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
@@ -22,5 +22,16 @@ package org.apache.slider.providers.agent;
public enum Command {
NOP, // do nothing
INSTALL, // Install the component
- START // Start the component
+ START; // Start the component
+
+ public static Command getCommand(String commandVal) {
+ if (commandVal.equals(Command.START.toString())) {
+ return Command.START;
+ }
+ if (commandVal.equals(Command.INSTALL.toString())) {
+ return Command.INSTALL;
+ }
+
+ return Command.NOP;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java b/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
index f318096..35d9116 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
@@ -22,5 +22,19 @@ package org.apache.slider.providers.agent;
public enum CommandResult {
IN_PROGRESS, // Command is in progress
COMPLETED, // Command has successfully completed
- FAILED // Command has failed
+ FAILED; // Command has failed
+
+ public static CommandResult getCommandResult(String commandResVal) {
+ if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
+ return CommandResult.COMPLETED;
+ }
+ if (commandResVal.equals(CommandResult.FAILED.toString())) {
+ return CommandResult.FAILED;
+ }
+ if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
+ return CommandResult.IN_PROGRESS;
+ }
+
+ throw new IllegalArgumentException("Unrecognized value " + commandResVal);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
index 2ad16af..60a6f82 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
@@ -37,6 +37,8 @@ public class ComponentInstanceState {
private State targetState = State.STARTED;
private int failuresSeen = 0;
private Boolean configReported = false;
+ private long lastHeartbeat = 0;
+ private ContainerState containerState;
public ComponentInstanceState(String compName,
String containerId,
@@ -44,6 +46,8 @@ public class ComponentInstanceState {
this.compName = compName;
this.containerId = containerId;
this.applicationId = applicationId;
+ this.containerState = ContainerState.INIT;
+ this.lastHeartbeat = System.currentTimeMillis();
}
public String getCompName() {
@@ -58,6 +62,26 @@ public class ComponentInstanceState {
this.configReported = configReported;
}
+ public ContainerState getContainerState() {
+ return containerState;
+ }
+
+ public void setContainerState(ContainerState containerState) {
+ this.containerState = containerState;
+ }
+
+ public long getLastHeartbeat() {
+ return lastHeartbeat;
+ }
+
+ public void setLastHeartbeat(long lastHeartbeat) {
+ this.lastHeartbeat = lastHeartbeat;
+ if(this.containerState == ContainerState.UNHEALTHY ||
+ this.containerState == ContainerState.INIT) {
+ this.containerState = ContainerState.HEALTHY;
+ }
+ }
+
public void commandIssued(Command command) {
Command expected = getNextCommand();
if (expected != command) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
new file mode 100644
index 0000000..0394ba2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+/** The states a component instance can be. */
+public enum ContainerState {
+ INIT, // Container is not net activated
+ HEALTHY, // Agent is heartbeating
+ UNHEALTHY, // Container is unhealthy - no heartbeat for some interval
+ HEARTBEAT_LOST; // Container is lost - request a new instance
+
+ /**
+ * Indicates whether or not it is a valid state to produce a command.
+ *
+ * @return true if command can be issued for this state.
+ */
+ public boolean canIssueCommands() {
+ switch (this) {
+ case HEALTHY:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
new file mode 100644
index 0000000..3aeff66
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/** Monitors the container state and heartbeats. */
+public class HeartbeatMonitor implements Runnable {
+ protected static final Logger log =
+ LoggerFactory.getLogger(HeartbeatMonitor.class);
+ private final int threadWakeupInterval; //1 minute
+ private final AgentProviderService provider;
+ private volatile boolean shouldRun = true;
+ private Thread monitorThread = null;
+
+ public HeartbeatMonitor(AgentProviderService provider, int threadWakeupInterval) {
+ this.provider = provider;
+ this.threadWakeupInterval = threadWakeupInterval;
+ }
+
+ public void shutdown() {
+ shouldRun = false;
+ }
+
+ public void start() {
+ log.info("Starting heartbeat monitor with interval {}", threadWakeupInterval);
+ monitorThread = new Thread(this);
+ monitorThread.start();
+ }
+
+ void join(long millis) throws InterruptedException {
+ if (isAlive()) {
+ monitorThread.join(millis);
+ }
+ }
+
+ public boolean isAlive() {
+ if (monitorThread != null) {
+ return monitorThread.isAlive();
+ }
+ return false;
+ }
+
+ @Override
+ public void run() {
+ while (shouldRun) {
+ try {
+ log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " +
+ "milliseconds");
+ Thread.sleep(threadWakeupInterval);
+ doWork();
+ } catch (InterruptedException ex) {
+ log.warn("Scheduler thread is interrupted going to stop", ex);
+ shouldRun = false;
+ } catch (Exception ex) {
+ log.warn("Exception received", ex);
+ } catch (Throwable t) {
+ log.warn("ERROR", t);
+ }
+ }
+ }
+
+ /**
+ * Every interval the current state of the container are checked. If the state is INIT or HEALTHY and no HB are
+ * received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not
+ * communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is
+ * declared unavailable
+ */
+ private void doWork() {
+ Map<String, ComponentInstanceState> componentStatuses = provider.getComponentStatuses();
+ if (componentStatuses != null) {
+ for (String containerLabel : componentStatuses.keySet()) {
+ ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel);
+ long timeSinceLastHeartbeat = System.currentTimeMillis() - componentInstanceState.getLastHeartbeat();
+
+ if (timeSinceLastHeartbeat > threadWakeupInterval) {
+ if (componentInstanceState.getContainerState() == ContainerState.HEALTHY ||
+ componentInstanceState.getContainerState() == ContainerState.INIT) {
+ componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
+ log.warn("Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
+ containerLabel, componentInstanceState.getLastHeartbeat(),
+ timeSinceLastHeartbeat);
+ continue;
+ }
+ if (componentInstanceState.getContainerState() == ContainerState.UNHEALTHY
+ && timeSinceLastHeartbeat > threadWakeupInterval * 2) {
+ componentInstanceState.setContainerState(ContainerState.HEARTBEAT_LOST);
+ log.warn("Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
+ containerLabel, componentInstanceState.getLastHeartbeat(),
+ timeSinceLastHeartbeat);
+ this.provider.releaseContainer(containerLabel);
+ continue;
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
new file mode 100644
index 0000000..287035f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/** Operations available to a provider from AppMaster */
+public interface AMViewForProviders {
+ /** Provider can ask AppMaster to release a specific container */
+ void refreshContainer(String containerId, boolean newHostIfPossible) throws SliderException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 605c826..0142028 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -150,7 +150,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ServiceStateChangeListener,
RoleKeys,
ProviderCompleted,
- ContainerStartOperation {
+ ContainerStartOperation,
+ AMViewForProviders {
protected static final Logger log =
LoggerFactory.getLogger(SliderAppMaster.class);
@@ -675,8 +676,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry);
- sliderAMProvider.bind(stateForProviders, registry);
+ providerService.bind(stateForProviders, registry, this);
+ sliderAMProvider.bind(stateForProviders, registry, null);
// now do the registration
registerServiceInstance(clustername, appid);
@@ -1345,6 +1346,30 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
+
+ /* =================================================================== */
+ /* ProviderAMOperations */
+ /* =================================================================== */
+
+ /**
+ * Refreshes the container by releasing it and having it reallocated
+ *
+ * @param containerId id of the container to release
+ * @param newHostIfPossible allocate the replacement container on a new host
+ *
+ * @throws SliderException
+ */
+ public void refreshContainer(String containerId, boolean newHostIfPossible)
+ throws SliderException {
+ log.info(
+ "Refreshing container {} per provider request.",
+ containerId);
+ rmOperationHandler.execute(appState.releaseContainer(containerId));
+
+ // ask for more containers if needed
+ reviewRequestAndReleaseNodes();
+ }
+
/* =================================================================== */
/* ServiceStateChangeListener */
/* =================================================================== */
@@ -1515,5 +1540,4 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//now have the service launcher do its work
ServiceLauncher.serviceMain(extendedArgs);
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index b5e67f5..9981f68 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1476,6 +1476,26 @@ public class AppState {
return operations;
}
+ /**
+ * Releases a container based on container id
+ * @param containerId
+ * @return
+ * @throws SliderInternalStateException
+ */
+ public List<AbstractRMOperation> releaseContainer(String containerId)
+ throws SliderInternalStateException {
+ List<AbstractRMOperation> operations = new ArrayList<>();
+ List<RoleInstance> activeRoleInstances = cloneActiveContainerList();
+ for (RoleInstance role : activeRoleInstances) {
+ if (role.container.getId().toString().equals(containerId)) {
+ containerReleaseSubmitted(role.container);
+ operations.add(new ContainerReleaseOperation(role.getId()));
+ }
+ }
+
+ return operations;
+ }
+
/**
* Find a container running on a specific host -looking
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
new file mode 100644
index 0000000..1f495c6
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json
@@ -0,0 +1,29 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "heartbeat.monitor.interval": "20000",
+ "agent.instance.debug.data": "ANY:DO_NOT_REGISTER:NONE",
+ "agent.conf": "agent.ini",
+ "application.def": "cmd_log_app_pkg.zip",
+ "config_types": "cl-site",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "package_list": "files/command_log_10.tar",
+ "site.global.app_user": "yarn",
+ "site.global.application_id": "CommandLogger",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-0.96.1-hadoop2",
+ "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
+ "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
+ "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
+ },
+ "components": {
+ "COMMAND_LOGGER": {
+ },
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
new file mode 100644
index 0000000..c824e03
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json
@@ -0,0 +1,29 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "heartbeat.monitor.interval": "20000",
+ "agent.instance.debug.data": "ANY:DO_NOT_HEARTBEAT:DO_NOT_HEARTBEAT:NONE",
+ "agent.conf": "agent.ini",
+ "application.def": "cmd_log_app_pkg.zip",
+ "config_types": "cl-site",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "package_list": "files/command_log_10.tar",
+ "site.global.app_user": "yarn",
+ "site.global.application_id": "CommandLogger",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-0.96.1-hadoop2",
+ "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
+ "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
+ "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
+ },
+ "components": {
+ "COMMAND_LOGGER": {
+ },
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 361fc2e..873bc93 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -34,6 +34,7 @@ import org.apache.slider.core.launch.ContainerLauncher
import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.AMViewForProviders
import org.apache.slider.server.appmaster.state.StateAccessForProviders
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations
import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat
@@ -194,7 +195,8 @@ class MockProviderService implements ProviderService {
@Override
void bind(
StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry) {
+ RegistryViewForProviders registry,
+ AMViewForProviders amView) {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
new file mode 100644
index 0000000..ec62b54
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ *
+ */
+public class TestAgentLaunchParameter {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentLaunchParameter.class);
+
+ @Test
+ public void testTestAgentLaunchParameter() throws Exception {
+ AgentLaunchParameter alp = new AgentLaunchParameter("");
+ Assert.assertEquals("", alp.getNextLaunchParameter("abc"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
+
+ alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE");
+ Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("2", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+
+ Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("6", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("c"));
+
+ alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:");
+ Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+
+ alp = new AgentLaunchParameter("|:");
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("a"));
+
+ alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:");
+ Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
+
+ alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::");
+ Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
new file mode 100644
index 0000000..c2cfafd
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+
+/**
+ *
+ */
+public class TestHeartbeatMonitor {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestHeartbeatMonitor.class);
+
+ @Test
+ public void testRegularHeartbeat() throws Exception {
+ AgentProviderService provider = createNiceMock(AgentProviderService.class);
+ HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 1 * 1000);
+ Assert.assertFalse(hbm.isAlive());
+ expect(provider.getComponentStatuses()).andReturn(null).anyTimes();
+ replay(provider);
+ hbm.start();
+ Assert.assertTrue(hbm.isAlive());
+ hbm.shutdown();
+ Thread.sleep(1 * 1000);
+ Assert.assertFalse(hbm.isAlive());
+ }
+
+ @Test
+ public void testHeartbeatMonitorWithHealthy() throws Exception {
+ AgentProviderService provider = createNiceMock(AgentProviderService.class);
+ HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 500);
+ Assert.assertFalse(hbm.isAlive());
+ Map<String, ComponentInstanceState> statuses = new HashMap<>();
+ ComponentInstanceState state = new ComponentInstanceState("HBASE_MASTER", "Cid", "Aid");
+ state.setState(State.STARTED);
+ state.setLastHeartbeat(System.currentTimeMillis());
+ statuses.put("label_1", state);
+ expect(provider.getComponentStatuses()).andReturn(statuses).anyTimes();
+ replay(provider);
+ hbm.start();
+ Assert.assertTrue(hbm.isAlive());
+ Thread.sleep(1 * 1000);
+ hbm.shutdown();
+ Thread.sleep(1 * 1000);
+ Assert.assertFalse(hbm.isAlive());
+ }
+
+ @Test
+ public void testHeartbeatMonitorWithUnhealthyAndThenLost() throws Exception {
+ AgentProviderService provider = createNiceMock(AgentProviderService.class);
+ HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 2 * 1000);
+ Assert.assertFalse(hbm.isAlive());
+ Map<String, ComponentInstanceState> statuses = new HashMap<>();
+ ComponentInstanceState masterState = new ComponentInstanceState("HBASE_MASTER", "Cid1", "Aid1");
+ masterState.setState(State.STARTED);
+ masterState.setLastHeartbeat(System.currentTimeMillis());
+ statuses.put("Aid1_Cid1_HBASE_MASTER", masterState);
+
+ ComponentInstanceState slaveState = new ComponentInstanceState("HBASE_REGIONSERVER", "Cid2", "Aid1");
+ slaveState.setState(State.STARTED);
+ slaveState.setLastHeartbeat(System.currentTimeMillis());
+ statuses.put("Aid1_Cid2_HBASE_REGIONSERVER", slaveState);
+
+ expect(provider.getComponentStatuses()).andReturn(statuses).anyTimes();
+ expect(provider.releaseContainer("Aid1_Cid2_HBASE_REGIONSERVER")).andReturn(true).once();
+ replay(provider);
+ hbm.start();
+
+ Thread.sleep(1 * 1000);
+ // just dial back by at least 2 sec but no more than 4
+ slaveState.setLastHeartbeat(System.currentTimeMillis() - (2 * 1000 + 100));
+ masterState.setLastHeartbeat(System.currentTimeMillis());
+
+ Thread.sleep(1 * 1000 + 500);
+ masterState.setLastHeartbeat(System.currentTimeMillis());
+
+ log.info("Slave container state {}", slaveState.getContainerState());
+ Assert.assertEquals(ContainerState.HEALTHY, masterState.getContainerState());
+ Assert.assertEquals(ContainerState.UNHEALTHY, slaveState.getContainerState());
+
+ Thread.sleep(1 * 1000);
+ // some lost heartbeats are ignored (e.g. ~ 1 sec)
+ masterState.setLastHeartbeat(System.currentTimeMillis() - 1 * 1000);
+
+ Thread.sleep(1 * 1000 + 500);
+
+ log.info("Slave container state {}", slaveState.getContainerState());
+ Assert.assertEquals(ContainerState.HEALTHY, masterState.getContainerState());
+ Assert.assertEquals(ContainerState.HEARTBEAT_LOST, slaveState.getContainerState());
+ hbm.shutdown();
+ }
+
+ @Test
+ public void testHeartbeatTransitions() {
+ ComponentInstanceState slaveState = new ComponentInstanceState("HBASE_REGIONSERVER", "Cid2", "Aid1");
+ slaveState.setState(State.STARTED);
+
+ Assert.assertEquals(ContainerState.INIT, slaveState.getContainerState());
+ slaveState.setLastHeartbeat(System.currentTimeMillis());
+ Assert.assertEquals(ContainerState.HEALTHY, slaveState.getContainerState());
+
+ slaveState.setContainerState(ContainerState.UNHEALTHY);
+ Assert.assertEquals(ContainerState.UNHEALTHY, slaveState.getContainerState());
+ slaveState.setLastHeartbeat(System.currentTimeMillis());
+ Assert.assertEquals(ContainerState.HEALTHY, slaveState.getContainerState());
+
+ slaveState.setContainerState(ContainerState.HEARTBEAT_LOST);
+ Assert.assertEquals(ContainerState.HEARTBEAT_LOST, slaveState.getContainerState());
+ slaveState.setLastHeartbeat(System.currentTimeMillis());
+ Assert.assertEquals(ContainerState.HEARTBEAT_LOST, slaveState.getContainerState());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 97199f4..2427009 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -17,6 +17,7 @@
package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.slider.providers.agent.AgentProviderService;
+import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.slf4j.Logger;
@@ -39,8 +40,8 @@ public class TestAgentProviderService extends AgentProviderService{
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg) {
- super.bind(stateAccessor, reg);
+ RegistryViewForProviders reg, AMViewForProviders amView) {
+ super.bind(stateAccessor, reg, amView);
Map<String,String> dummyProps = new HashMap<>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
index d5a91f0..6333fad 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
+import org.apache.hadoop.security.AccessControlException
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
@@ -28,22 +29,17 @@ import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.apache.tools.zip.ZipEntry
import org.apache.tools.zip.ZipOutputStream
-import org.junit.Assert
-import org.junit.Assume
import org.junit.Before
import org.junit.BeforeClass
-import org.junit.rules.TemporaryFolder
import org.junit.Rule
-
-import org.apache.hadoop.security.AccessControlException
-
+import org.junit.rules.TemporaryFolder
@Slf4j
class AgentCommandTestBase extends CommandTestBase
- implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
public static final boolean AGENTTESTS_ENABLED
-
+
protected static String APP_RESOURCE = "../slider-core/src/test/app_packages/test_command_log/resources.json"
protected static String APP_TEMPLATE = "../slider-core/src/test/app_packages/test_command_log/appConfig.json"
protected static String APP_PKG_DIR = "../slider-core/src/test/app_packages/test_command_log/"
@@ -85,7 +81,7 @@ class AgentCommandTestBase extends CommandTestBase
} catch (AccessControlException ace) {
log.info "No write access to test user home directory. " +
"Ensure home directory exists and has correct permissions." + ace.getMessage()
- Assume.assumeTrue("Ensure home directory exists and has correct permissions for test user.", false)
+ fail("Ensure home directory exists and has correct permissions for test user.")
}
}
@@ -135,11 +131,11 @@ class AgentCommandTestBase extends CommandTestBase
assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
}
- public static String findLineEntry(SliderShell shell, String[] locators) {
+ public static String findLineEntry(SliderShell shell, String[] locaters) {
int index = 0;
for (String str in shell.out) {
- if (str.contains("\"" + locators[index] + "\"")) {
- if (locators.size() == index + 1) {
+ if (str.contains("\"" + locaters[index] + "\"")) {
+ if (locaters.size() == index + 1) {
return str;
} else {
index++;
@@ -150,8 +146,29 @@ class AgentCommandTestBase extends CommandTestBase
return null;
}
- public static boolean isAppRunning(String text, SliderShell shell) {
+ public static String findLineEntryValue(SliderShell shell, String[] locaters) {
+ String line = findLineEntry(shell, locaters);
+
+ if (line != null) {
+ log.info("Parsing {} for value.", line)
+ int dividerIndex = line.indexOf(":");
+ if (dividerIndex > 0) {
+ String value = line.substring(dividerIndex + 1).trim()
+ if (value.endsWith(",")) {
+ value = value.subSequence(0, value.length() - 1)
+ }
+ return value;
+ }
+ }
+ return null;
+ }
+
+ public static boolean isApplicationInState(String text, String applicationName) {
boolean exists = false
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_LIST,
+ applicationName])
for (String str in shell.out) {
if (str.contains(text)) {
exists = true
@@ -161,23 +178,14 @@ class AgentCommandTestBase extends CommandTestBase
return exists
}
- protected static void ensureApplicationIsUp(String clusterName, int maxAttemptCount = 15) {
- SliderShell shell
- int attemptCount = 0
- while (attemptCount < maxAttemptCount) {
- shell = slider(EXIT_SUCCESS, [
- ACTION_LIST,
- clusterName])
-
- if (isAppRunning("RUNNING", shell)) {
- break
- }
-
- attemptCount++
- assert attemptCount != maxAttemptCount, 'Application did not start, aborting test.'
+ protected void ensureApplicationIsUp(String clusterName) {
+ repeatUntilTrue(this.&isApplicationUp, 15, 1000 * 3, ['arg1': clusterName],
+ true, 'Application did not start, aborting test.')
+ }
- sleep(1000 * 3)
- }
+ boolean isApplicationUp(Map<String, String> args) {
+ String applicationName = args['arg1'];
+ return isApplicationInState("RUNNING", applicationName);
}
public static void addDir(File dirObj, ZipOutputStream zipFile, String prefix) {
@@ -204,4 +212,38 @@ class AgentCommandTestBase extends CommandTestBase
out.close();
}
}
+
+ protected void repeatUntilTrue(Closure c, int maxAttempts, int sleepDur, Map args,
+ boolean failIfUnsuccessful = false, String message = "") {
+ int attemptCount = 0
+ while (attemptCount < maxAttempts) {
+ if (c(args)) {
+ break
+ };
+ attemptCount++;
+
+ if (failIfUnsuccessful) {
+ assert attemptCount != maxAttempts, message
+ }
+
+ sleep(sleepDur)
+ }
+ }
+
+ protected void cleanup(String applicationName) throws Throwable {
+ log.info "Cleaning app instance, if exists, by name " + applicationName
+ teardown(applicationName)
+
+ // sleep till the instance is frozen
+ sleep(1000 * 3)
+
+ SliderShell shell = slider([
+ ACTION_DESTROY,
+ applicationName])
+
+ if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
+ logShell(shell)
+ assert fail("Old cluster either should not exist or should get destroyed.")
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy
index 3d0b270..06809cc 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentClusterLifecycle.groovy
@@ -51,7 +51,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@After
public void destroyCluster() {
- teardown(CLUSTER)
+ cleanup(CLUSTER)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures.groovy
new file mode 100644
index 0000000..96a99ad
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures.groovy
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestAgentFailures extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "one-container-fail-register"
+ private static String APP_TEMPLATE2 =
+ "../slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json"
+
+
+ @After
+ public void destroyCluster() {
+ cleanup(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testAgentFailRegistrationOnce() throws Throwable {
+ if (!AGENTTESTS_ENABLED) {
+ log.info "TESTS are not run."
+ return
+ }
+
+ cleanup(APPLICATION_NAME)
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_IMAGE, agentTarballPath.toString(),
+ ARG_TEMPLATE, APP_TEMPLATE2,
+ ARG_RESOURCES, APP_RESOURCE
+ ])
+
+ logShell(shell)
+
+ ensureApplicationIsUp(APPLICATION_NAME)
+
+ repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '2']);
+
+ sleep(1000 * 20)
+
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ assertComponentCount(COMMAND_LOGGER, 1, shell)
+ String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ assert requested != null && requested.isInteger() && requested.toInteger() >= 2,
+ 'At least 2 containers must be requested'
+
+ assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+
+ assertSuccess(shell)
+ }
+
+
+ boolean hasContainerCountExceeded(Map<String, String> args) {
+ int expectedCount = args['arg1'].toInteger();
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ //logShell(shell)
+ String requested = findLineEntryValue(
+ shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
+ return true
+ }
+
+ return false
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures2.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures2.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures2.groovy
new file mode 100644
index 0000000..644fa4f
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAgentFailures2.groovy
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class TestAgentFailures2 extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "two-container-fail-heartbeat"
+ private static String APP_TEMPLATE3 =
+ "../slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json"
+
+
+ @After
+ public void destroyCluster() {
+ cleanup(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testAgentFailHeartbeatingTwiceOnce() throws Throwable {
+ if (!AGENTTESTS_ENABLED) {
+ log.info "TESTS are not run."
+ return
+ }
+
+ cleanup(APPLICATION_NAME)
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_IMAGE, agentTarballPath.toString(),
+ ARG_TEMPLATE, APP_TEMPLATE3,
+ ARG_RESOURCES, APP_RESOURCE
+ ])
+
+ logShell(shell)
+
+ ensureApplicationIsUp(APPLICATION_NAME)
+
+ repeatUntilTrue(this.&hasContainerCountExceeded, 20, 1000 * 10, ['arg1': '3']);
+
+ sleep(1000 * 20)
+
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ assertComponentCount(COMMAND_LOGGER, 1, shell)
+ String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ assert requested != null && requested.isInteger() && requested.toInteger() >= 3,
+ 'At least 2 containers must be requested'
+
+ assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+
+ assertSuccess(shell)
+ }
+
+
+ boolean hasContainerCountExceeded(Map<String, String> args) {
+ int expectedCount = args['arg1'].toInteger();
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ //logShell(shell)
+ String requested = findLineEntryValue(
+ shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
+ return true
+ }
+
+ return false
+ }
+}
[28/50] [abbrv] git commit: Modify README.txt for HBase app package
Posted by st...@apache.org.
Modify README.txt for HBase app package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3be6c869
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3be6c869
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3be6c869
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 3be6c86922d0413163e65766d827bfcc03d60f88
Parents: 209cee4
Author: tedyu <yu...@gmail.com>
Authored: Tue Jun 24 14:00:22 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue Jun 24 14:00:22 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/README.txt | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3be6c869/app-packages/hbase/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt
index fb84750..78d033e 100644
--- a/app-packages/hbase/README.txt
+++ b/app-packages/hbase/README.txt
@@ -17,10 +17,17 @@
How to create a Slider package for HBase?
+The version of HBase used for the app package can be adjusted by adding a
+flag such as
+ -Dhbase.version=0.98.3
+
Copy the tarball for HBase:
cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files/
-If, HBase tarball is publised in maven repository, you can use:
+Use the following command to install HBase tarball locally:
+ mvn install:install-file -Dfile=<path-to-tarball> -DgroupId=org.apache.hbase -DartifactId=hbase -Dversion=0.98.3-hadoop2 -Dclassifier=bin -Dpackaging=tar.gz
+
+After HBase tarball is publised locally in maven repository, you can use the following command:
mvn clean package -DskipTests -Phbase-app-package
App package can be found in
app-packages/HBase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip
@@ -36,10 +43,6 @@ work well as the default configuration for Slider apps. So it is advisable that
when you create an application package for Slider, include sample/default
resources.json and appConfig.json for a minimal Yarn cluster.
-The version of HBase used for the app package can be adjusted by adding a
-flag such as
- -Dhbase.version=0.98.3
-
If an HBase version older than 0.98.3 is desired, it must be installed in the
local maven repo.
[08/50] [abbrv] git commit: SLIDER-157 Remove app-packages/hbase-v0_96
Posted by st...@apache.org.
SLIDER-157 Remove app-packages/hbase-v0_96
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bc0c6728
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bc0c6728
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bc0c6728
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: bc0c672843efaa8d83a97a94d1152f6ebd1c27c1
Parents: 7729653
Author: tedyu <yu...@gmail.com>
Authored: Mon Jun 23 09:24:46 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Jun 23 09:24:46 2014 -0700
----------------------------------------------------------------------
.../slider_specs/creating_app_definitions.md | 80 +++++++++++---------
1 file changed, 43 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc0c6728/src/site/markdown/slider_specs/creating_app_definitions.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/creating_app_definitions.md b/src/site/markdown/slider_specs/creating_app_definitions.md
index be3dd33..ab9f51c 100644
--- a/src/site/markdown/slider_specs/creating_app_definitions.md
+++ b/src/site/markdown/slider_specs/creating_app_definitions.md
@@ -70,45 +70,51 @@ Additional configuration parameters are described in `app-packages/accumulo/READ
For other app packages, follow this example:
-* cd slider/app-packages/hbase-v0_96
-* zip -r hbase_v096.zip .
-* Looking at the content through unzip -l "$@" hbase_v096.zip
+* cd slider/app-packages/hbase
+* zip -r hbase-v098.zip .
+* Looking at the content through unzip -l "$@" hbase-v098.zip
```
-Archive: hbase_v096.zip
- Length Date Time Name
- -------- ---- ---- ----
- 3163 05-16-14 16:32 appConfig.json
- 0 05-02-14 07:51 configuration/
- 5077 05-02-14 07:51 configuration/global.xml
- 5248 05-02-14 07:51 configuration/hbase-log4j.xml
- 2250 05-02-14 07:51 configuration/hbase-policy.xml
- 14705 05-02-14 07:51 configuration/hbase-site.xml
- 3332 05-16-14 16:33 metainfo.xml
- 0 05-02-14 07:51 package/
- 0 05-19-14 20:36 package/files/
- 83154798 05-19-14 20:36 package/files/hbase-0.96.1-hadoop2-bin.tar.gz
- 0 05-02-14 07:51 package/scripts/
- 787 05-02-14 07:51 package/scripts/__init__.py
- 1378 05-02-14 07:51 package/scripts/functions.py
- 3599 05-02-14 07:51 package/scripts/hbase.py
- 1205 05-02-14 07:51 package/scripts/hbase_client.py
- 1640 05-02-14 07:51 package/scripts/hbase_master.py
- 1764 05-02-14 07:51 package/scripts/hbase_regionserver.py
- 1482 05-02-14 07:51 package/scripts/hbase_service.py
- 4924 05-02-14 07:51 package/scripts/params.py
- 973 05-02-14 07:51 package/scripts/status_params.py
- 0 05-02-14 07:51 package/templates/
- 2723 05-02-14 07:51 package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
- 2723 05-02-14 07:51 package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
- 3878 05-02-14 07:51 package/templates/hbase-env.sh.j2
- 909 05-02-14 07:51 package/templates/hbase_client_jaas.conf.j2
- 989 05-02-14 07:51 package/templates/hbase_master_jaas.conf.j2
- 1001 05-02-14 07:51 package/templates/hbase_regionserver_jaas.conf.j2
- 837 05-02-14 07:51 package/templates/regionservers.j2
- 357 05-12-14 12:04 resources.json
- -------- -------
- 83219742 29 files
+Archive: hbase-v098.zip
+ Length Date Time Name
+--------- ---------- ----- ----
+ 3158 06-19-2014 22:08 appConfig.json
+ 0 06-19-2014 22:15 configuration/
+ 5077 06-19-2014 22:15 configuration/global.xml
+ 5248 06-19-2014 22:15 configuration/hbase-log4j.xml
+ 2250 06-19-2014 22:15 configuration/hbase-policy.xml
+ 14705 06-19-2014 22:15 configuration/hbase-site.xml
+ 2142 06-19-2014 22:17 jmx_metrics.json
+ 3991 06-19-2014 22:45 metainfo.xml
+ 0 06-19-2014 22:33 package/
+ 0 06-19-2014 22:31 package/files/
+ 83278591 06-19-2014 22:31 package/files/hbase-0.98.3-hadoop2-bin.tar.gz
+ 0 06-19-2014 22:29 package/scripts/
+ 787 06-19-2014 22:27 package/scripts/__init__.py
+ 1378 06-19-2014 22:27 package/scripts/functions.py
+ 3738 06-19-2014 22:27 package/scripts/hbase.py
+ 1205 06-19-2014 22:27 package/scripts/hbase_client.py
+ 1640 06-19-2014 22:27 package/scripts/hbase_master.py
+ 1764 06-19-2014 22:27 package/scripts/hbase_regionserver.py
+ 1482 06-19-2014 22:27 package/scripts/hbase_service.py
+ 5089 06-19-2014 22:27 package/scripts/params.py
+ 973 06-19-2014 22:27 package/scripts/status_params.py
+ 0 06-19-2014 22:34 package/templates/
+ 2799 06-19-2014 22:34 package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
+ 2799 06-19-2014 22:34 package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
+ 3878 06-19-2014 22:34 package/templates/hbase-env.sh.j2
+ 909 06-19-2014 22:34 package/templates/hbase_client_jaas.conf.j2
+ 989 06-19-2014 22:34 package/templates/hbase_master_jaas.conf.j2
+ 1001 06-19-2014 22:34 package/templates/hbase_regionserver_jaas.conf.j2
+ 837 06-19-2014 22:34 package/templates/regionservers.j2
+ 3486 06-19-2014 22:39 pom.xml
+ 1921 06-19-2014 21:53 README.txt
+ 358 06-19-2014 22:36 resources.json
+ 0 06-19-2014 22:40 src/
+ 0 06-19-2014 23:02 src/assembly/
+ 2400 06-19-2014 22:45 src/assembly/hbase.xml
+--------- -------
+ 83354595 35 files
```
Sample **resources.json** and **appConfig.json** files are also included in the enlistment. These are samples and are typically tested on one node test installations.
[19/50] [abbrv] SLIDER-121 removed site documentation from git source
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/specification/cli-actions.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/specification/cli-actions.md b/src/site/markdown/specification/cli-actions.md
deleted file mode 100644
index 5060ab5..0000000
--- a/src/site/markdown/specification/cli-actions.md
+++ /dev/null
@@ -1,675 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider CLI Actions
-
-
-## Important
-
-1. This document is still being updated from the original hoya design
-2. The new cluster model of separated specification files for internal, resource and application configuration
-has not been incorporated.
-1. What is up to date is the CLI command list and arguments
-
-## client configuration
-
-As well as the CLI options, the `conf/slider-client.xml` XML file can define arguments used to communicate with the Application instance
-
-
-#### `fs.defaultFS`
-
-Equivalent to setting the filesystem with `--filesystem`
-
-
-
-## Common
-
-### System Properties
-
-Arguments of the form `-S key=value` define JVM system properties.
-
-These are supported primarily to define options needed for some Kerberos configurations.
-
-### Definitions
-
-Arguments of the form `-D key=value` define JVM system properties.
-
-These can define client options that are not set in `conf/slider-client.xml` - or to override them.
-
-### Cluster names
-
-All actions that must take an instance name will fail with `EXIT_UNKNOWN_INSTANCE`
-if one is not provided.
-
-## Action: Build
-
-Builds a cluster -creates all the on-filesystem datastructures, and generates a cluster description
-that is both well-defined and deployable -*but does not actually start the cluster*
-
- build (instancename,
- options:List[(String,String)],
- components:List[(String, int)],
- componentOptions:List[(String,String, String)],
- resourceOptions:List[(String,String)],
- resourceComponentOptions:List[(String,String, String)],
- confdir: URI,
- provider: String
- zkhosts,
- zkport,
- image
- apphome
- appconfdir
-
-
-#### Preconditions
-
-(Note that the ordering of these preconditions is not guaranteed to remain constant)
-
-The instance name is valid
-
- if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
-
-The instance must not be live. This is purely a safety check as the next test should have the same effect.
-
- if slider-instance-live(YARN, instancename) : raise SliderException(EXIT_CLUSTER_IN_USE)
-
-The instance must not exist
-
- if is-dir(HDFS, instance-path(FS, instancename)) : raise SliderException(EXIT_CLUSTER_EXISTS)
-
-The configuration directory must exist it does not have to be the instance's HDFS instance,
-as it will be copied there -and must contain only files
-
- let FS = FileSystem.get(appconfdir)
- if not isDir(FS, appconfdir) raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
- forall f in children(FS, appconfdir) :
- if not isFile(f): raise IOException
-
-There's a race condition at build time where between the preconditions being met and the instance specification being saved, the instance
-is created by another process. This addressed by creating a lock file, `writelock.json` in the destination directory. If the file
-exists, no other process may acquire the lock.
-
-There is a less exclusive readlock file, `readlock.json` which may be created by any process that wishes to read the configuration.
-If it exists when another process wishes to access the files, the subsequent process may read the data, but MUST NOT delete it
-afterwards. A process attempting to acquire the writelock must check for the existence of this file before AND after creating the
-writelock file, failing if its present. This retains a small race condition: a second or later reader may still be reading the data
-when a process successfully acquires the write lock. If this proves to be an issue, a stricter model could be implemented, with each reading process creating a unique named readlock- file.
-
-
-
-
-#### Postconditions
-
-All the instance directories exist
-
- is-dir(HDFS', instance-path(HDFS', instancename))
- is-dir(HDFS', original-conf-path(HDFS', instancename))
- is-dir(HDFS', generated-conf-path(HDFS', instancename))
-
-The application cluster specification saved is well-defined and deployable
-
- let instance-description = parse(data(HDFS', instance-json-path(HDFS', instancename)))
- well-defined-instance(instance-description)
- deployable-application-instance(HDFS', instance-description)
-
-More precisely: the specification generated before it is saved as JSON is well-defined and deployable; no JSON file will be created
-if the validation fails.
-
-Fields in the cluster description have been filled in
-
- internal.global["internal.provider.name"] == provider
- app_conf.global["zookeeper.port"] == zkport
- app_conf.global["zookeeper.hosts"] == zkhosts
-
-
- package => app_conf.global["agent.package"] = package
-
-
-
-Any `apphome` and `image` properties have propagated
-
- apphome == null or clusterspec.options["cluster.application.home"] == apphome
- image == null or clusterspec.options["cluster.application.image.path"] == image
-
-(The `well-defined-application-instance()` requirement above defines the valid states
-of this pair of options)
-
-
-All role sizes have been mapped to `component.instances` fields
-
- forall (name, size) in components :
- resources.components[name]["components.instances"] == size
-
-
-
-
-All option parameters have been added to the `options` map in the specification
-
- forall (opt, val) in options :
- app_conf.global[opt] == val
-
- forall (opt, val) in resourceOptions :
- resource.global[opt] == val
-
-All component option parameters have been added to the specific components's option map
-in the relevant configuration file
-
- forall (name, opt, val) in componentOptions :
- app_conf.components[name][opt] == val
-
- forall (name, opt, val) in resourceComponentOptions :
- resourceComponentOptions.components[name][opt] == val
-
-To avoid some confusion as to where keys go, all options beginning with the
-prefix `component.` are automatically copied into the resources file:
-
- forall (opt, val) in options where startswith(opt, "component.")
- or startswith(opt, "role.")
- or startswith(opt, "yarn."):
- resource.global[opt] == val
-
- forall (name, opt, val) in componentOptions where startswith(opt, "component.")
- or startswith(opt, "role.")
- or startswith(opt, "yarn."):
- resourceComponentOptions.components[name][opt] == val
-
-
-There's no explicit rejection of duplicate options, the outcome of that
-state is 'undefined'.
-
-What is defined is that if Slider or its provider provided a default option value,
-the command-line supplied option will override it.
-
-All files that were in the configuration directory are now copied into the "original" configuration directory
-
- let FS = FileSystem.get(appconfdir)
- let dest = original-conf-path(HDFS', instancename)
- forall [c in children(FS, confdir) :
- data(HDFS', dest + [filename(c)]) == data(FS, c)
-
-All files that were in the configuration directory now have equivalents in the generated configuration directory
-
- let FS = FileSystem.get(appconfdir)
- let dest = generated-conf-path(HDFS', instancename)
- forall [c in children(FS, confdir) :
- isfile(HDFS', dest + [filename(c)])
-
-
-## Action: Thaw
-
- thaw <instancename> [--wait <timeout>]
-
-Thaw takes an application instance with configuration and (possibly) data on disk, and
-attempts to create a live application with the specified number of nodes
-
-#### Preconditions
-
- if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
-
-The cluster must not be live. This is purely a safety check as the next test should have the same effect.
-
- if slider-instance-live(YARN, instancename) : raise SliderException(EXIT_CLUSTER_IN_USE)
-
-The cluster must not exist
-
- if is-dir(HDFS, application-instance-path(FS, instancename)) : raise SliderException(EXIT_CLUSTER_EXISTS)
-
-The cluster specification must exist, be valid and deployable
-
- if not is-file(HDFS, cluster-json-path(HDFS, instancename)) : SliderException(EXIT_UNKNOWN_INSTANCE)
- if not well-defined-application-instance(HDFS, application-instance-path(HDFS, instancename)) : raise SliderException(EXIT_BAD_CLUSTER_STATE)
- if not deployable-application-instance(HDFS, application-instance-path(HDFS, instancename)) : raise SliderException(EXIT_BAD_CLUSTER_STATE)
-
-### Postconditions
-
-
-After the thaw has been performed, there is now a queued request in YARN
-for the chosen (how?) queue
-
- YARN'.Queues'[amqueue] = YARN.Queues[amqueue] + [launch("slider", instancename, requirements, context)]
-
-If a wait timeout was specified, the cli waits until the application is considered
-running by YARN (the AM is running), the wait timeout has been reached, or
-the application has failed
-
- waittime < 0 or (exists a in slider-running-application-instances(yarn-application-instances(YARN', instancename, user))
- where a.YarnApplicationState == RUNNING)
-
-
-## Outcome: AM-launched state
-
-Some time after the AM was queued, if the relevant
-prerequisites of the launch request are met, the AM will be deployed
-
-#### Preconditions
-
-* The resources referenced in HDFS (still) are accessible by the user
-* The requested YARN memory and core requirements could be met on the YARN cluster and
-specific YARN application queue.
-* There is sufficient capacity in the YARN cluster to create a container for the AM.
-
-#### Postconditions
-
-Define a YARN state at a specific time `t` as `YARN(t)`; the fact that
-an AM is launched afterwards
-
-The AM is deployed if there is some time `t` after the submission time `t0`
-where the application is listed
-
- exists t1 where t1 > t0 and slider-instance-live(YARN(t1), user, instancename)
-
-At which time there is a container in the cluster hosting the AM -it's
-context is the launch context
-
- exists c in containers(YARN(t1)) where container.context = launch.context
-
-There's no way to determine when this time `t1` will be reached -or if it ever
-will -its launch may be postponed due to a lack of resources and/or higher priority
-requests using resources as they become available.
-
-For tests on a dedicated YARN cluster, a few tens of seconds appear to be enough
-for the AM-launched state to be reached, a failure to occur, or to conclude
-that the resource requirements are unsatisfiable.
-
-## Outcome: AM-started state
-
-A (usually short) time after the AM is launched, it should start
-
-* The node hosting the container is working reliably
-* The supplied command line could start the process
-* the localized resources in the context could be copied to the container (which implies
-that they are readable by the user account the AM is running under)
-* The combined classpath of YARN, extra JAR files included in the launch context,
-and the resources in the slider client 'conf' dir contain all necessary dependencies
-to run Slider.
-* There's no issue with the cluster specification that causes the AM to exit
-with an error code.
-
-Node failures/command line failures are treated by YARN as an AM failure which
-will trigger a restart attempt -this may be on the same or a different node.
-
-#### preconditions
-
-The AM was launched at an earlier time, `t1`
-
- exists t1 where t1 > t0 and am-launched(YARN(t1)
-
-
-#### Postconditions
-
-The application is actually started if it is listed in the YARN application list
-as being in the state `RUNNING`, an RPC port has been registered with YARN (visible as the `rpcPort`
-attribute in the YARN Application Report,and that port is servicing RPC requests
-from authenticated callers.
-
- exists t2 where:
- t2 > t1
- and slider-instance-live(YARN(t2), YARN, instancename, user)
- and slider-live-instances(YARN(t2))[0].rpcPort != 0
- and rpc-connection(slider-live-instances(YARN(t2))[0], SliderClusterProtocol)
-
-A test for accepting cluster requests is querying the cluster status
-with `SliderClusterProtocol.getJSONClusterStatus()`. If this returns
-a parseable cluster description, the AM considers itself live.
-
-## Outcome: Applicaton Instance operational state
-
-Once started, Slider enters the operational state of trying to keep the numbers
-of live role instances matching the numbers specified in the cluster specification.
-
-The AM must request the a container for each desired instance of a specific roles of the
-application, wait for those requests to be granted, and then instantiate
-the specific application roles on the allocated containers.
-
-Such a request is made on startup, whenever a failure occurs, or when the
-cluster size is dynamically updated.
-
-The AM releases containers when the cluster size is shrunk during a flex operation,
-or during teardown.
-
-### steady state condition
-
-The steady state of a Slider cluster is that the number of live instances of a role,
-plus the number of requested instances , minus the number of instances for
-which release requests have been made must match that of the desired number.
-
-If the internal state of the Slider AM is defined as `AppState`
-
- forall r in clusterspec.roles :
- r["yarn.component.instances"] ==
- AppState.Roles[r].live + AppState.Roles[r].requested - AppState.Roles[r].released
-
-The `AppState` represents Slider's view of the external YARN system state, based on its
-history of notifications received from YARN.
-
-It is indirectly observable from the cluster state which an AM can be queried for
-
-
- forall r in AM.getJSONClusterStatus().roles :
- r["yarn.component.instances"] ==
- r["role.actual.instances"] + r["role.requested.instances"] - r["role.releasing.instances"]
-
-Slider does not consider it an error if the number of actual instances remains below
-the desired value (i.e. outstanding requests are not being satisfied) -this is
-an operational state of the cluster that Slider cannot address.
-
-### Cluster startup
-
-On a healthy dedicated test cluster, the time for the requests to be satisfied is
-a few tens of seconds at most: a failure to achieve this state is a sign of a problem.
-
-### Node or process failure
-
-After a container or node failure, a new container for a new instance of that role
-is requested.
-
-The failure count is incremented -it can be accessed via the `"role.failed.instances"`
-attribute of a role in the status report.
-
-The number of failures of a role is tracked, and used by Slider as to when to
-conclude that the role is somehow failing consistently -and it should fail the
-entire application.
-
-This has initially been implemented as a simple counter, with the cluster
-option: `"slider.container.failure.threshold"` defining that threshold.
-
- let status = AM.getJSONClusterStatus()
- forall r in in status.roles :
- r["role.failed.instances"] < status.options["slider.container.failure.threshold"]
-
-
-### Instance startup failure
-
-
-Startup failures are measured alongside general node failures.
-
-A container is deemed to have failed to start if either of the following conditions
-were met:
-
-1. The AM received an `onNodeManagerContainerStartFailed` event.
-
-1. The AM received an `onCompletedNode` event on a node that started less than
-a specified number of seconds earlier -a number given in the cluster option
-`"slider.container.failure.shortlife"`.
-
-More sophisticated failure handling logic than is currently implemented may treat
-startup failures differently from ongoing failures -as they can usually be
-treated as a sign that the container is failing to launch the program reliably -
-either the generated command line is invalid, or the application is failing
-to run/exiting on or nearly immediately.
-
-## Action: Create
-
-Create is simply `build` + `thaw` in sequence - the postconditions from the first
-action are intended to match the preconditions of the second.
-
-## Action: Freeze
-
- freeze instancename [--wait time] [--message message]
-
-The *freeze* action "freezes" the cluster: all its nodes running in the YARN
-cluster are stopped, leaving all the persistent state.
-
-The operation is intended to be idempotent: it is not an error if
-freeze is invoked on an already frozen cluster
-
-#### Preconditions
-
-The cluster name is valid and it matches a known cluster
-
- if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
-
- if not is-file(HDFS, application-instance-path(HDFS, instancename)) :
- raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
-#### Postconditions
-
-If the cluster was running, an RPC call has been sent to it `stopCluster(message)`
-
-If the `--wait` argument specified a wait time, then the command will block
-until the cluster has finished or the wait time was exceeded.
-
-If the `--message` argument specified a message -it must appear in the
-YARN logs as the reason the cluster was frozen.
-
-
-The outcome should be the same:
-
- not slider-instance-live(YARN', instancename)
-
-## Action: Flex
-
-Flex the cluster size: add or remove roles.
-
- flex instancename
- components:List[(String, int)]
-
-1. The JSON cluster specification in the filesystem is updated
-1. if the cluster is running, it is given the new cluster specification,
-which will change the desired steady-state of the application
-
-#### Preconditions
-
- if not is-file(HDFS, cluster-json-path(HDFS, instancename)) :
- raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
-#### Postconditions
-
- let originalSpec = data(HDFS, cluster-json-path(HDFS, instancename))
-
- let updatedSpec = originalspec where:
- forall (name, size) in components :
- updatedSpec.roles[name]["yarn.component.instances"] == size
- data(HDFS', cluster-json-path(HDFS', instancename)) == updatedSpec
- rpc-connection(slider-live-instances(YARN(t2))[0], SliderClusterProtocol)
- let flexed = rpc-connection(slider-live-instances(YARN(t2))[0], SliderClusterProtocol).flexClusterupdatedSpec)
-
-
-#### AM actions on flex
-
- boolean SliderAppMaster.flexCluster(ClusterDescription updatedSpec)
-
-If the cluster is in a state where flexing is possible (i.e. it is not in teardown),
-then `AppState` is updated with the new desired role counts. The operation will
-return once all requests to add or remove role instances have been queued,
-and be `True` iff the desired steady state of the cluster has been changed.
-
-#### Preconditions
-
- well-defined-application-instance(HDFS, updatedSpec)
-
-
-#### Postconditions
-
- forall role in AppState.Roles.keys:
- AppState'.Roles'[role].desiredCount = updatedSpec[roles]["yarn.component.instances"]
- result = AppState' != AppState
-
-
-The flexing may change the desired steady state of the cluster, in which
-case the relevant requests will have been queued by the completion of the
-action. It is not possible to state whether or when the requests will be
-satisfied.
-
-## Action: Destroy
-
-Idempotent operation to destroy a frozen cluster -it succeeds if the
-cluster has already been destroyed/is unknown, but not if it is
-actually running.
-
-#### Preconditions
-
- if not valid-instance-name(instancename) : raise SliderException(EXIT_COMMAND_ARGUMENT_ERROR)
-
- if slider-instance-live(YARN, instancename) : raise SliderException(EXIT_CLUSTER_IN_USE)
-
-
-#### Postconditions
-
-The cluster directory and all its children do not exist
-
- not is-dir(HDFS', application-instance-path(HDFS', instancename))
-
-
-## Action: Status
-
- status instancename [--out outfile]
- 2
-#### Preconditions
-
- if not slider-instance-live(YARN, instancename) : raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
-#### Postconditions
-
-The status of the application has been successfully queried and printed out:
-
- let status = slider-live-instances(YARN).rpcPort.getJSONClusterStatus()
-
-if the `outfile` value is not defined then the status appears part of stdout
-
- status in STDOUT'
-
-otherwise, the outfile exists in the local filesystem
-
- (outfile != "") ==> data(LocalFS', outfile) == body
- (outfile != "") ==> body in STDOUT'
-
-## Action: Exists
-
-This probes for a named cluster being defined or actually being in the running
-state.
-
-In the running state; it is essentially the status
-operation with only the exit code returned
-
-#### Preconditions
-
-
- if not is-file(HDFS, application-instance-path(HDFS, instancename)) :
- raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
-#### Postconditions
-
-The operation succeeds if the cluster is running and the RPC call returns the cluster
-status.
-
- if live and not slider-instance-live(YARN, instancename):
- retcode = -1
- else:
- retcode = 0
-
-## Action: getConf
-
-This returns the live client configuration of the cluster -the
-site-xml file.
-
- getconf --format (xml|properties) --out [outfile]
-
-*We may want to think hard about whether this is needed*
-
-#### Preconditions
-
- if not slider-instance-live(YARN, instancename) : raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
-
-#### Postconditions
-
-The operation succeeds if the cluster status can be retrieved and saved to
-the named file/printed to stdout in the format chosen
-
- let status = slider-live-instances(YARN).rpcPort.getJSONClusterStatus()
- let conf = status.clientProperties
- if format == "xml" :
- let body = status.clientProperties.asXmlDocument()
- else:
- let body = status.clientProperties.asProperties()
-
- if outfile != "" :
- data(LocalFS', outfile) == body
- else
- body in STDOUT'
-
-## Action: list
-
- list [instancename]
-
-Lists all clusters of a user, or only the one given
-
-#### Preconditions
-
-If a instancename is specified it must be in YARNs list of active or completed applications
-of that user:
-
- if instancename != "" and [] == yarn-application-instances(YARN, instancename, user)
- raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
-
-#### Postconditions
-
-If no instancename was given, all slider applications of that user are listed,
-else only the one running (or one of the finished ones)
-
- if instancename == "" :
- forall a in yarn-application-instances(YARN, user) :
- a.toString() in STDOUT'
- else
- let e = yarn-application-instances(YARN, instancename, user)
- e.toString() in STDOUT'
-
-## Action: killcontainer
-
-This is an operation added for testing. It will kill a container in the cluster
-*without flexing the cluster size*. As a result, the cluster will detect the
-failure and attempt to recover from the failure by instantiating a new instance
-of the cluster
-
- killcontainer cluster --id container-id
-
-#### Preconditions
-
- if not slider-instance-live(YARN, instancename) : raise SliderException(EXIT_UNKNOWN_INSTANCE)
-
- exists c in slider-app-containers(YARN, instancename, user) where c.id == container-id
-
- let status := AM.getJSONClusterStatus()
- exists role = status.instances where container-id in status.instances[role].values
-
-
-#### Postconditions
-
-The container is not in the list of containers in the cluster
-
- not exists c in containers(YARN) where c.id == container-id
-
-And implicitly, not in the running containers of that application
-
- not exists c in slider-app-containers(YARN', instancename, user) where c.id == container-id
-
-At some time `t1 > t`, the status of the application (`AM'`) will be updated to reflect
-that YARN has notified the AM of the loss of the container
-
-
- let status' = AM'.getJSONClusterStatus()
- len(status'.instances[role]) < len(status.instances[role])
- status'.roles[role]["role.failed.instances"] == status'.roles[role]["role.failed.instances"]+1
-
-
-At some time `t2 > t1` in the future, the size of the containers of the application
-in the YARN cluster `YARN''` will be as before
-
- let status'' = AM''.getJSONClusterStatus()
- len(status''.instances[r] == len(status.instances[r])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/specification/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/specification/index.md b/src/site/markdown/specification/index.md
deleted file mode 100644
index f4c8d67..0000000
--- a/src/site/markdown/specification/index.md
+++ /dev/null
@@ -1,41 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Specification of Apache Slider behaviour
-
-This is a a "more rigorous" definition of the behavior of Slider in terms
-of its state and its command-line operations -by defining a 'formal' model
-of HDFS, YARN and Slider's internal state, then describing the operations
-that can take place in terms of their preconditions and postconditions.
-
-This is to show what tests we can create to verify that an action
-with a valid set of preconditions results in an outcome whose postconditions
-can be verified. It also makes more apparent what conditions should be
-expected to result in failures, as well as what the failure codes should be.
-
-Specifying the behavior has also helped identify areas where there was ambiguity,
-where clarification and more tests were needed.
-
-The specification depends on ongoing work in [HADOOP-9361](https://issues.apache.org/jira/browse/HADOOP-9361):
-to define the Hadoop Filesytem APIs --This specification uses [the same notation](https://github.com/steveloughran/hadoop-trunk/blob/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md)
-
-
-1. [Model: YARN And Slider](slider-model.html)
-1. [CLI actions](cli-actions.html)
-
-Exceptions and operations may specify exit codes -these are listed in
-[Client Exit Codes](../exitcodes.html)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/specification/slider-model.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/specification/slider-model.md b/src/site/markdown/specification/slider-model.md
deleted file mode 100644
index 75f8c68..0000000
--- a/src/site/markdown/specification/slider-model.md
+++ /dev/null
@@ -1,286 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Formal Apache Slider Model
-
-This is the model of Slider and YARN for the rest of the specification.
-
-## File System
-
-A File System `HDFS` represents a Hadoop FileSystem -either HDFS or another File
-System which spans the cluster. There are also other filesystems that
-can act as sources of data that is then copied into HDFS. These will be marked
-as `FS` or with the generic `FileSystem` type.
-
-
-There's ongoing work in [HADOOP-9361](https://issues.apache.org/jira/browse/HADOOP-9361)
-to define the Hadoop Filesytem APIs using the same notation as here,
-the latest version being available on [github](https://github.com/steveloughran/hadoop-trunk/tree/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem)
-Two key references are
-
- 1. [The notation reused in the Slider specifications](https://github.com/steveloughran/hadoop-trunk/blob/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/notation.md)
- 1. [The model of the filesystem](https://github.com/steveloughran/hadoop-trunk/blob/stevel/HADOOP-9361-filesystem-contract/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/model.md)
-
- The model and its predicates and invariants will be used in these specifications.
-
-## YARN
-
-From the perspective of YARN application, The YARN runtime is a state, `YARN`,
-comprised of: ` (Apps, Queues, Nodes)`
-
- Apps: Map[AppId, ApplicationReport]
-
-An application has a name, an application report and a list of outstanding requests
-
- App: (Name, report: ApplicationReport, Requests:List[AmRequest])
-
-An application report contains a mixture of static and dynamic state of the application
-and the AM.
-
- ApplicationReport: AppId, Type, User, YarnApplicationState, AmContainer, RpcPort, TrackingURL,
-
-YARN applications have a number of states. These are ordered such that if the
-`state.ordinal() > RUNNING.ordinal() ` then the application has entered an exit state.
-
- YarnApplicationState : [NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED ]
-
-AMs can request containers to be added or released
-
- AmRequest = { add-container(priority, requirements), release(containerId)}
-
-Job queues are named queues of job requests; there is always a queue called `"default"`
-
- Queues: Map[String:Queue]
- Queue: List[Requests]
- Request = {
- launch(app-name, app-type, requirements, context)
- }
- Context: (localized-resources: Map[String,URL], command)
-
-
-This is doesn't completely model the cluster from the AM perspective -there's no
-notion of node operations (launching code in a container) or events coming from YARN.
-
-The `Nodes` structure models the nodes in a cluster
-
- Nodes: Map[nodeID,(name, containers:List[Container])]
-
-A container contains some state
-
- Container: (containerId, appId, context)
-
-The containers in a cluster are the aggregate set of all containers across
-all nodes
-
- def containers(YARN) =
- [c for n in keys(YARN.Nodes) for c in YARN.Nodes[n].Containers ]
-
-
-The containers of an application are all containers that are considered owned by it,
-
- def app-containers(YARN, appId: AppId) =
- [c in containers(YARN) where c.appId == appId ]
-
-### Operations & predicates used the specifications
-
-
- def applications(YARN, type) =
- [ app.report for app in YARN.Apps.values where app.report.Type == type]
-
- def user-applications(YARN, type, user)
- [a in applications(YARN, type) where: a.User == user]
-
-
-## UserGroupInformation
-
-Applications are launched and executed on hosts computers: either client machines
-or nodes in the cluster, these have their own state which may need modeling
-
- HostState: Map[String, String]
-
-A key part of the host state is actually the identity of the current user,
-which is used to define the location of the persistent state of the cluster -including
-its data, and the identity under which a deployed container executes.
-
-In a secure cluster, this identity is accompanied by kerberos tokens that grant the caller
-access to the filesystem and to parts of YARN itself.
-
-This specification does not currently explicitly model the username and credentials.
-If it did they would be used throughout the specification to bind to a YARN or HDFS instance.
-
-`UserGroupInformation.getCurrentUser(): UserGroupInformation`
-
-Returns the current user information. This information is immutable and fixed for the duration of the process.
-
-
-
-## Slider Model
-
-### Cluster name
-
-A valid cluster name is a name of length > 1 which follows the internet hostname scheme of letter followed by letter or digit
-
- def valid-cluster-name(c) =
- len(c)> 0
- and c[0] in ['a'..'z']
- and c[1] in (['a'..'z'] + ['-'] + ['0..9'])
-
-### Persistent Cluster State
-
-A Slider cluster's persistent state is stored in a path
-
- def cluster-path(FS, clustername) = user-home(FS) + ["clusters", clustername]
- def cluster-json-path(FS, clustername) = cluster-path(FS, clustername) + ["cluster.json"]
- def original-conf-path(FS, clustername) = cluster-path(FS, clustername) + ["original"]
- def generated-conf-path(FS, clustername) = cluster-path(FS, clustername) + ["generated"]
- def data-path(FS, clustername) = cluster-path(FS, clustername) + ["data"]
-
-When a cluster is built/created the specified original configuration directory
-is copied to `original-conf-path(FS, clustername)`; this is patched for the
-specific instance bindings and saved into `generated-conf-path(FS, clustername)`.
-
-A cluster *exists* if all of these paths are found:
-
- def cluster-exists(FS, clustername) =
- is-dir(FS, cluster-path(FS, clustername))
- and is-file(FS, cluster-json-path(FS, clustername))
- and is-dir(FS, original-conf-path(FS, clustername))
- and generated-conf-path(FS, original-conf-path(FS, clustername))
-
-A cluster is considered `running` if there is a Slider application type belonging to the current user in one of the states
-`{NEW, NEW_SAVING, SUBMITTED, ACCEPTED, RUNNING}`.
-
- def final-yarn-states = {FINISHED, FAILED, KILLED }
-
- def slider-app-instances(YARN, clustername, user) =
- [a in user-applications(YARN, "slider", user) where:
- and a.Name == clustername]
-
- def slider-app-running-instances(YARN, clustername, user) =
- [a in slider-app-instances(YARN, user, clustername) where:
- not a.YarnApplicationState in final-yarn-state]
-
- def slider-app-running(YARN, clustername, user) =
- [] != slider-app-running-instances(YARN, clustername, user)
-
- def slider-app-live-instances(YARN, clustername, user) =
- [a in slider-app-instances(YARN, user, clustername) where:
- a.YarnApplicationState == RUNNING]
-
- def slider-app-live(YARN, clustername, user) =
- [] != slider-app-live-instances(YARN, clustername, user)
-
-### Invariant: there must never be more than one running instance of a named Slider cluster
-
-
-There must never be more than one instance of the same Slider cluster running:
-
- forall a in user-applications(YARN, "slider", user):
- len(slider-app-running-instances(YARN, a.Name, user)) <= 1
-
-There may be multiple instances in a finished state, and one running instance alongside multiple finished instances -the applications
-that work with Slider MUST select a running cluster ahead of any terminated clusters.
-
-### Containers of an application
-
-
-The containers of a slider application are the set of containers of that application
-
- def slider-app-containers(YARN, clustername, user) =
- app-containers(YARN, appid where
- appid = slider-app-running-instances(YARN, clustername, user)[0])
-
-
-
-
-### RPC Access to a slider cluster
-
-
- An application is accepting RPC requests for a given protocol if there is a port binding
- defined and it is possible to authenticate a connection using the specified protocol
-
- def rpc-connection(appReport, protocol) =
- appReport.host != null
- appReport.rpcPort != 0
- and RPC.getProtocolProxy(appReport.host, appReport.rpcPort, protocol)
-
- Being able to open an RPC port is the strongest definition of liveness possible
- to make: if the AM responds to RPC operations, it is doing useful work.
-
-### Valid Cluster Description
-
-The `cluster.json` file of a cluster configures Slider to deploy the application.
-
-#### well-defined-cluster(cluster-description)
-
-A Cluster Description is well-defined if it is valid JSON and required properties are present
-
-**OBSOLETE**
-
-
-Irrespective of specific details for deploying the Slider AM or any provider-specific role instances,
-a Cluster Description defined in a `cluster.json` file at the path `cluster-json-path(FS, clustername)`
-is well-defined if
-
-1. It is parseable by the jackson JSON parser.
-1. Root elements required of a Slider cluster specification must be defined, and, where appropriate, non-empty
-1. It contains the extensible elements required of a Slider cluster specification. For example, `options` and `roles`
-1. The types of the extensible elements match those expected by Slider.
-1. The `version` element matches a supported version
-1. Exactly one of `options/cluster.application.home` and `options/cluster.application.image.path` must exist.
-1. Any cluster options that are required to be integers must be integers
-
-This specification is very vague here to avoid duplication: the cluster description structure is currently implicitly defined in
-`org.apache.slider.api.ClusterDescription`
-
-Currently Slider ignores unknown elements during parsing. This may be changed.
-
-The test for this state does not refer to the cluster filesystem
-
-#### deployable-cluster(FS, cluster-description)
-
-A Cluster Description defines a deployable cluster if it is well-defined cluster and the contents contain valid information to deploy a cluster
-
-This defines how a cluster description is valid in the extends the valid configuration with
-
-* The entry `name` must match a supported provider
-* Any elements that name the cluster match the cluster name as defined by the path to the cluster:
-
- originConfigurationPath == original-conf-path(FS, clustername)
- generatedConfigurationPath == generated-conf-path(FS, clustername)
- dataPath == data-path(FS, clustername)
-
-* The paths defined in `originConfigurationPath` , `generatedConfigurationPath` and `dataPath` must all exist.
-* `options/zookeeper.path` must be defined and refer to a path in the ZK cluster
-defined by (`options/zookeeper.hosts`, `zookeeper.port)` to which the user has write access (required by HBase and Accumulo)
-* If `options/cluster.application.image.path` is defined, it must exist and be readable by the user.
-* It must declare a type that maps to a provider entry in the Slider client's XML configuration:
-
- len(clusterspec["type"]) > 0
- clientconfig["slider.provider."+ clusterspec["type"]] != null
-
-* That entry must map to a class on the classpath which can be instantiated
-and cast to `SliderProviderFactory`.
-
- let classname = clientconfig["slider.provider."+ clusterspec["type"]]
- (Class.forName(classname).newInstance()) instanceof SliderProviderFactory
-
-#### valid-for-provider(cluster-description, provider)
-
-A provider considers a specification valid if its own validation logic is satisfied. This normally
-consists of rules about the number of instances of different roles; it may include other logic.
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/troubleshooting.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/troubleshooting.md b/src/site/markdown/troubleshooting.md
deleted file mode 100644
index 42bef8e..0000000
--- a/src/site/markdown/troubleshooting.md
+++ /dev/null
@@ -1,154 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Troubleshooting
-
-Slider can be tricky to start using, because it combines the need to set
-up a YARN application, with the need to have an HBase configuration
-that works
-
-
-### Common problems
-
-## Classpath for Slider AM wrong
-
-The Slider Application Master, the "Slider AM" builds up its classpath from
-those JARs it has locally, and the JARS pre-installed on the classpath
-
-This often surfaces in an exception that can be summarized as
-"hadoop-common.jar is not on the classpath":
-
- Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/util/ExitUtil$ExitException
- Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.util.ExitUtil$ExitException
- at java.net.URLClassLoader$1.run(URLClassLoader.java:202)
- at java.security.AccessController.doPrivileged(Native Method)
- at java.net.URLClassLoader.findClass(URLClassLoader.java:190)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:306)
- at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:301)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:247)
- Could not find the main class: org.apache.hadoop.yarn.service.launcher.ServiceLauncher. Program will exit.
-
-
-For ambari-managed deployments, we recommend the following
-
-
- <property>
- <name>yarn.application.classpath</name>
- <value>
- /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
- </value>
- </property>
-
-The `yarn-site.xml` file for the site will contain the relevant value.
-
-### Application Instantiation fails, "TriggerClusterTeardownException: Unstable Cluster"
-
-Slider gives up if it cannot keep enough instances of a role running -or more
-precisely, if they keep failing.
-
-If this happens on cluster startup, it means that the application is not working
-
- org.apache.slider.core.exceptions.TriggerClusterTeardownException: Unstable Cluster:
- - failed with role worker failing 4 times (4 in startup); threshold is 2
- - last failure: Failure container_1386872971874_0001_01_000006 on host 192.168.1.86,
- see http://hor12n22.gq1.ygridcore.net:19888/jobhistory/logs/192.168.1.86:45454/container_1386872971874_0001_01_000006/ctx/yarn
-
-This message warns that a role -here worker- is failing to start and it has failed
-more than the configured failure threshold is. What it doesn't do is say why it failed,
-because that is not something the AM knows -that is a fact hidden in the logs on
-the container that failed.
-
-The final bit of the exception message can help you track down the problem,
-as it points you to the logs.
-
-In the example above the failure was in `container_1386872971874_0001_01_000006`
-on the host `192.168.1.86`. If you go to then node manager on that machine (the YARN
-RM web page will let you do this), and look for that container,
-you may be able to grab the logs from it.
-
-A quicker way is to browse to the URL on the next line.
-Note: the URL depends on yarn.log.server.url being properly configured.
-
-It is from those logs that the cause of the problem -because they are the actual
-output of the actual application which Slider is trying to deploy.
-
-
-
-### Not all the containers start -but whenever you kill one, another one comes up.
-
-This is often caused by YARN not having enough capacity in the cluster to start
-up the requested set of containers. The AM has submitted a list of container
-requests to YARN, but only when an existing container is released or killed
-is one of the outstanding requests granted.
-
-Fix #1: Ask for smaller containers
-
-edit the `yarn.memory` option for roles to be smaller: set it 64 for a smaller
-YARN allocation. *This does not affect the actual heap size of the
-application component deployed*
-
-Fix #2: Tell YARN to be less strict about memory consumption
-
-Here are the properties in `yarn-site.xml` which we set to allow YARN
-to schedule more role instances than it nominally has room for.
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1</value>
- </property>
- <property>
- <description>Whether physical memory limits will be enforced for
- containers.
- </description>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
- <!-- we really don't want checking here-->
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
-If you create too many instances, your hosts will start swapping and
-performance will collapse -we do not recommend using this in production.
-
-
-### Configuring YARN for better debugging
-
-
-One configuration to aid debugging is tell the nodemanagers to
-keep data for a short period after containers finish
-
- <!-- 10 minutes after a failure to see what is left in the directory-->
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>600</value>
- </property>
-
-You can then retrieve logs by either the web UI, or by connecting to the
-server (usually by `ssh`) and retrieve the logs from the log directory
-
-
-We also recommend making sure that YARN kills processes
-
- <!--time before the process gets a -9 -->
- <property>
- <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
- <value>30000</value>
- </property>
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/hoya_am_architecture.png
----------------------------------------------------------------------
diff --git a/src/site/resources/hoya_am_architecture.png b/src/site/resources/hoya_am_architecture.png
deleted file mode 100644
index 191a8db..0000000
Binary files a/src/site/resources/hoya_am_architecture.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/app_config_folders_01.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/app_config_folders_01.png b/src/site/resources/images/app_config_folders_01.png
deleted file mode 100644
index 4e78b63..0000000
Binary files a/src/site/resources/images/app_config_folders_01.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/app_package_sample_04.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/app_package_sample_04.png b/src/site/resources/images/app_package_sample_04.png
deleted file mode 100644
index 170256b..0000000
Binary files a/src/site/resources/images/app_package_sample_04.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/image_0.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/image_0.png b/src/site/resources/images/image_0.png
deleted file mode 100644
index e62a3e7..0000000
Binary files a/src/site/resources/images/image_0.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/image_1.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/image_1.png b/src/site/resources/images/image_1.png
deleted file mode 100644
index d0888ac..0000000
Binary files a/src/site/resources/images/image_1.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/managed_client.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/managed_client.png b/src/site/resources/images/managed_client.png
deleted file mode 100644
index 9c094b1..0000000
Binary files a/src/site/resources/images/managed_client.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/slider-container.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/slider-container.png b/src/site/resources/images/slider-container.png
deleted file mode 100644
index 2e02833..0000000
Binary files a/src/site/resources/images/slider-container.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/resources/images/unmanaged_client.png
----------------------------------------------------------------------
diff --git a/src/site/resources/images/unmanaged_client.png b/src/site/resources/images/unmanaged_client.png
deleted file mode 100644
index 739d56d..0000000
Binary files a/src/site/resources/images/unmanaged_client.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/site.xml
----------------------------------------------------------------------
diff --git a/src/site/site.xml b/src/site/site.xml
deleted file mode 100644
index 12dc5cf..0000000
--- a/src/site/site.xml
+++ /dev/null
@@ -1,84 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project name="Apache Slider ${project.version} (incubating)">
-<!--
-
- <skin>
- <groupId>org.apache.maven.skins</groupId>
- <artifactId>maven-stylus-skin</artifactId>
- <version>1.2</version>
- </skin>
-
- <skin>
- <groupId>org.apache.maven.skins</groupId>
- <artifactId>maven-application-skin</artifactId>
- <version>1.0</version>
- </skin>
-
--->
-
- <skin>
- <groupId>org.apache.maven.skins</groupId>
- <artifactId>maven-fluido-skin</artifactId>
- <version>1.3.0</version>
- </skin>
-
- <custom>
- <fluidoSkin>
- <topBarEnabled>true</topBarEnabled>
- <sideBarEnabled>false</sideBarEnabled>
- </fluidoSkin>
- </custom>
-
- <version position="right"/>
-
- <bannerLeft>
- <name>Apache Slider (incubating)</name>
- <href>http://slider.incubator.apache.org</href>
- </bannerLeft>
-
- <bannerRight>
- <src>http://incubator.apache.org/images/apache-incubator-logo.png</src>
- </bannerRight>
-
- <body>
-
- <menu ref="reports"/>
-
- <menu name="Documents">
- <item name="Getting Started" href="/getting_started.html"/>
- <item name="manpage" href="/manpage.html"/>
- <item name="Troubleshooting" href="/troubleshooting.html"/>
- <item name="Architecture" href="/architecture/index.html"/>
- <item name="Developing" href="/developing/index.html"/>
- <item name="Exitcodes" href="/exitcodes.html"/>
- </menu>
-
- <menu name="ASF">
- <item name="How Apache Works" href="http://www.apache.org/foundation/how-it-works.html"/>
- <item name="Developer Documentation" href="http://www.apache.org/dev/"/>
- <item name="Foundation" href="http://www.apache.org/foundation/"/>
- <item name="Sponsor Apache" href="http://www.apache.org/foundation/sponsorship.html"/>
- <item name="Thanks" href="http://www.apache.org/foundation/thanks.html"/>
- </menu>
-
- <footer>
- <div class="row-fluid">Apache Slider, Slider, Apache, and the Apache Incubator logo are trademarks of The Apache Software Foundation.</div>
- </footer>
- </body>
-</project>
[40/50] [abbrv] git commit: SLIDER-174 HBase Master needs data
locality
Posted by st...@apache.org.
SLIDER-174 HBase Master needs data locality
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e4c855ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e4c855ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e4c855ad
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: e4c855ad8ea095e3304b161b0a3df87a2b4e93cc
Parents: 22c9689
Author: tedyu <yu...@gmail.com>
Authored: Thu Jun 26 09:27:25 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jun 26 09:27:25 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/providers/hbase/HBaseRoles.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e4c855ad/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
index 552374c..01776f7 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -42,8 +42,7 @@ public class HBaseRoles {
*/
static {
ROLES.add(new ProviderRole(HBaseKeys.ROLE_WORKER, KEY_WORKER));
- // Master doesn't need data locality
- ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER,PlacementPolicy.NO_DATA_LOCALITY));
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER));
}
[15/50] [abbrv] git commit: SLIDER-160: initial client settings for
an ambari vagrant VM up on c6401
Posted by st...@apache.org.
SLIDER-160: initial client settings for an ambari vagrant VM up on c6401
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/81578df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/81578df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/81578df2
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 81578df22f2eee22f49419e48a2f3e07c33005ff
Parents: 153f5e9
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 17:35:23 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 17:35:23 2014 -0700
----------------------------------------------------------------------
src/test/clusters/c6401/slider/log4j.properties | 83 ++++++++++++++++++++
.../clusters/c6401/slider/slider-client.xml | 71 +++++++++++++++++
2 files changed, 154 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/81578df2/src/test/clusters/c6401/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/log4j.properties b/src/test/clusters/c6401/slider/log4j.properties
new file mode 100644
index 0000000..6211771
--- /dev/null
+++ b/src/test/clusters/c6401/slider/log4j.properties
@@ -0,0 +1,83 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/81578df2/src/test/clusters/c6401/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/slider-client.xml b/src/test/clusters/c6401/slider/slider-client.xml
new file mode 100644
index 0000000..f1aa4fb
--- /dev/null
+++ b/src/test/clusters/c6401/slider/slider-client.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM
+-->
+<configuration>
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>configs/c6401/slider</value>
+ <description>This is just for diagnostics</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>c6401:8050</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://c6401.ambari.apache.org:8020</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>c6401:2181</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>
+ /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+ </value>
+ </property>
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.am.restart.time</name>
+ <description>Time in millis to await an AM restart</description>
+ <value>60000</value>
+ </property>
+
+
+</configuration>
[49/50] [abbrv] git commit: SLIDER-153 refactor .py script slightly
for use elsewhere
Posted by st...@apache.org.
SLIDER-153 refactor .py script slightly for use elsewhere
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0ad4fb4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0ad4fb4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0ad4fb4f
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 0ad4fb4f7b5cc4e590cbb9e51aa7cefb59a0ae2d
Parents: a048c9f
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 30 16:16:08 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 30 16:16:08 2014 +0100
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 107 ++++++++++++++----------
1 file changed, 64 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0ad4fb4f/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index d48eca6..e60bed3 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -60,10 +60,10 @@ def confDir(sliderdir):
localconf = os.path.join(sliderdir, CONF)
return os.environ.get(SLIDER_CONF_DIR,localconf)
-def dirMustExist(dir):
- if not os.path.exists(dir):
- raise Exception("Directory does not exist: %s " % dir)
- return dir
+def dirMustExist(dirname):
+ if not os.path.exists(dirname):
+ raise Exception("Directory does not exist: %s " % dirname)
+ return dirname
def read(pipe, line):
"""
@@ -83,7 +83,61 @@ def read(pipe, line):
return line, True
else:
return line, False
-
+
+
+def runProcess(commandline):
+ """
+ Run a process
+ :param commandline: command line
+ :return:the return code
+ """
+ print "ready to exec : %s" % commandline
+ exe = subprocess.Popen(commandline,
+ stdin=None,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=False)
+ stdout = exe.stdout
+ stderr = exe.stderr
+ outline = ""
+ errline = ""
+ while exe.poll() is None:
+ # process is running; grab output and echo every line
+ outline, done = read(stdout, outline)
+ if done:
+ print outline
+ outline = ""
+ errline, done = read(stderr, errline)
+ if done:
+ print errline
+ errline = ""
+
+ # get tail
+ out, err = exe.communicate()
+ print outline + out.decode()
+ print errline + err.decode()
+ return exe.returncode
+
+
+def java(classname, args, classpath, jvm_opts_list):
+ """
+ Execute a java process, hooking up stdout and stderr
+ and printing them a line at a time as they come in
+ :param classname: classname
+ :param args: arguments to the java program
+ :param classpath: classpath
+ :param jvm_opts_list: list of JVM options
+ :return: the exit code.
+ """
+ # split the JVM opts by space
+ # java = "/usr/bin/java"
+ commandline = ["java"]
+ commandline.extend(jvm_opts_list)
+ commandline.append("-classpath")
+ commandline.append(classpath)
+ commandline.append(classname)
+ commandline.extend(args)
+ return runProcess(commandline)
def usage():
@@ -98,14 +152,13 @@ def main():
"""
if len(sys.argv)==1 :
return usage()
- print "stdout encoding: "+ sys.stdout.encoding
+ # print "stdout encoding: "+ sys.stdout.encoding
args = sys.argv[1:]
slider_home = sliderDir()
libdir = dirMustExist(libDir(slider_home))
confdir = dirMustExist(confDir(slider_home))
default_jvm_opts = DEFAULT_JVM__OPTS % confdir
slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
- # split the JVM opts by space
jvm_opts_split = slider_jvm_opts.split()
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
p = os.pathsep # path separator
@@ -119,42 +172,10 @@ def main():
print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
print "slider_classpath = \"%s\"" % slider_classpath
- #java = "/usr/bin/java"
- commandline = ["java", ]
- commandline.append("-classpath")
- commandline.append(slider_classpath)
- commandline.extend(jvm_opts_split)
- commandline.append(SLIDER_CLASSNAME)
- commandline.extend(args)
- print "ready to exec : %s" % commandline
- # docs warn of using PIPE on stderr
- exe = subprocess.Popen(commandline,
- stdin=None,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
- while exe.poll() is None:
- # process is running; grab output and echo every line
- outline, done = read(stdout, outline)
- if done:
- print outline
- outline = ""
- errline, done = read(stderr, errline)
- if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
- return exe.returncode
-
-
+ return java(SLIDER_CLASSNAME,
+ args,
+ slider_classpath,
+ jvm_opts_split)
if __name__ == '__main__':
"""
[06/50] [abbrv] SLIDER-157 Remove app-packages/hbase-v0_96
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7729653f/app-packages/hbase/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase/README.txt b/app-packages/hbase/README.txt
index 941579d..fb84750 100644
--- a/app-packages/hbase/README.txt
+++ b/app-packages/hbase/README.txt
@@ -17,11 +17,17 @@
How to create a Slider package for HBase?
- mvn clean package -DskipTests -Phbase-app-package
+Copy the tarball for HBase:
+ cp ~/Downloads/hbase-0.98.3-hadoop2-bin.tar.gz package/files/
+If, HBase tarball is publised in maven repository, you can use:
+ mvn clean package -DskipTests -Phbase-app-package
App package can be found in
app-packages/HBase/target/apache-slider-hbase-${hbase.version}-app-package-${slider.version}.zip
+Create a zip package at the root of the package (<slider enlistment>/app-packages/hbase/)
+ zip -r hbase-v098.zip .
+
Verify the content using
zip -Tv apache-slider-hbase-*.zip
[47/50] [abbrv] git commit: SLIDER-173 Container requests for the
same role should utilize different nodes - add more logs
Posted by st...@apache.org.
SLIDER-173 Container requests for the same role should utilize different nodes - add more logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/581c182b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/581c182b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/581c182b
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 581c182beccc24bbc720b363ba42ea927b09e84f
Parents: 736a9c7
Author: tedyu <yu...@gmail.com>
Authored: Fri Jun 27 09:29:32 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Jun 27 09:29:32 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/state/RoleHistory.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/581c182b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 990d735..0cd2b39 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -406,6 +406,7 @@ public class RoleHistory {
for (int i = 0; i < roleSize; i++) {
NodeEntry nodeEntry = ni.get(i);
if (nodeEntry != null && nodeEntry.isAvailable()) {
+ log.debug("Adding {} for role {}", ni, i);
getOrCreateNodesForRoleId(i).add(ni);
}
}
@@ -470,7 +471,7 @@ public class RoleHistory {
List<NodeInstance> targets = getNodesForRoleId(roleKey);
int cnt = targets == null ? 0 : targets.size();
- Log.info("There're " + cnt + " nodes to consider for " + role.getName());
+ log.info("There're {} nodes to consider for {}", cnt, role.getName());
while (targets != null && !targets.isEmpty() && nodeInstance == null) {
NodeInstance head = targets.remove(0);
if (head.getActiveRoleInstances(roleKey) == 0) {
@@ -478,7 +479,7 @@ public class RoleHistory {
}
}
if (nodeInstance == null) {
- Log.debug("No node selected for " + role.getName());
+ log.debug("No node selected for {}", role.getName());
}
return nodeInstance;
}
@@ -597,6 +598,7 @@ public class RoleHistory {
hosts = outstandingRequests.cancelOutstandingRequests(role);
if (!hosts.isEmpty()) {
//add the list
+ log.debug("Adding {} hosts for role {}", hosts.size(), role);
getOrCreateNodesForRoleId(role).addAll(hosts);
sortAvailableNodeList(role);
}
[21/50] [abbrv] SLIDER-121 removed site documentation from git source
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/manpage.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/manpage.md b/src/site/markdown/manpage.md
deleted file mode 100644
index 351fcc6..0000000
--- a/src/site/markdown/manpage.md
+++ /dev/null
@@ -1,483 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: YARN-hosted applications
-
-## NAME
-
-slider -YARN-hosted applications
-
-## SYNOPSIS
-
-Slider enables applications to be dynamically created on a YARN-managed datacenter.
-The program can be used to create, pause, and shutdown the application. It can also be used to list current active
-and existing but not running "frozen" application instances.
-
-## CONCEPTS
-
-1. A *Slider application* is an application packaged to be deployed by Slider. It consists of one or more distributed *components*
-
-1. A *Slider application instance* is a slider application configured to be deployable on a specific YARN cluster, with a specific configuration. An instance can be *live* -actually running- or *frozen*. When frozen all its configuration details and instance-specific data are preserved on HDFS.
-
-1. An *image* is a *tar.gz* file containing binaries used to create the application. 1. Images are kept in the HDFS filesystem and identified by their path names; filesystem permissions can be used to share images amongst users.
-
-1. An *image configuration* is a directory that is overlaid file-by-file onto the conf/ directory inside the HBase image.
-
-1. Users can have multiple image configurations -they too are kept in HDFS, identified by their path names, and can be shared by setting the appropriate permissions, along with a configuration template file.
-
-1. Only those files provided in the image configuration directory overwrite the default values contained in the image; all other configuration files are retained.
-
-1. Late-binding properties can also be provided at create time.
-
-1. Slider can overwrite some of the configuration properties to enable the dynamically created components to bind correctly to each other.
-
-1. An *instance directory* is a directory created in HDFS describing the application instance; it records the configuration -both user specified, application-default and any dynamically created by slider.
-
-1. A user can create an application instance.
-
-1. A live instances can be *frozen*, saving its final state to its application instance state directory. All running components are shut down.
-
-1. A frozen instance can be *thawed* -a its components started on or near the servers where they were previously running.
-
-1. A frozen instance can be *destroyed*.
-
-1. Running instances can be listed.
-
-1. An instance consists of a set of components
-
-1. The supported component types depends upon the slider application.
-
-1. the count of each component must initially be specified when a application instance is created.
-
-1. Users can flex an application instance: adding or removing components dynamically.
-If the application instance is live, the changes will have immediate effect. If not, the changes will be picked up when the instance is next thawed.
-
-
-<!--- ======================================================================= -->
-
-## Invoking Slider
-
-
- slider <ACTION> [<name>] [<OPTIONS>]
-
-
-<!--- ======================================================================= -->
-
-## COMMON COMMAND-LINE OPTIONS
-
-### `--conf configuration.xml`
-
-Configure the Slider client. This allows the filesystem, zookeeper instance and other properties to be picked up from the configuration file, rather than on the command line.
-
-Important: *this configuration file is not propagated to the application. It is purely for configuring the client itself.
-
-### `-D name=value`
-
-Define a Hadoop configuration option which overrides any options in the client configuration XML files.
-
-
-### `-m, --manager url`
-
-URL of the YARN resource manager
-
-
-### `--fs filesystem-uri`
-
-Use the specific filesystem URI as an argument to the operation.
-
-
-
-
-
-<!--- ======================================================================= -->
-
-
-<!--- ======================================================================= -->
-
-## Actions
-
-COMMANDS
-
-
-
-### `build <name>`
-
-Build an instance of the given name, with the specific options.
-
-It is not started; this can be done later with a `thaw` command.
-
-### `create <name>`
-
-Build and run an applicaton instance of the given name
-
-The `--wait` parameter, if provided, specifies the time to wait until the YARN application is actually running. Even after the YARN application has started, there may be some delay for the instance to start up.
-
-### Arguments for `build` and `create`
-
-
-
-##### `--package <uri-to-package>`
-
-This define the slider application package to be deployed.
-
-
-##### `--option <name> <value>`
-
-Set a application instance option.
-
-Example:
-
-Set an option to be passed into the `-site.xml` file of the target system, reducing
-the HDFS replication factor to 2. (
-
- --option site.dfs.blocksize 128m
-
-Increase the number of YARN containers which must fail before the Slider application instance
-itself fails.
-
- -O slider.container.failure.threshold 16
-
-##### `--appconf dfspath`
-
-A URI path to the configuration directory containing the template application specification. The path must be on a filesystem visible to all nodes in the YARN cluster.
-
-1. Only one configuration directory can be specified.
-1. The contents of the directory will only be read when the application instance is created/built.
-
-Example:
-
- --appconf hdfs://namenode/users/slider/conf/hbase-template
- --appconf file://users/accumulo/conf/template
-
-
-
-##### `--apphome localpath`
-
-A path to the home dir of a pre-installed application. If set when a Slider
-application instance is created, the instance will run with the binaries pre-installed
-on the nodes at this location
-
-*Important*: this is a path in the local filesystem which must be present
-on all hosts in the YARN cluster
-
-Example
-
- --apphome /usr/hadoop/hbase
-
-##### `--template <filename>`
-
-Filename for the template application instance configuration. This
-will be merged with -and can overwrite- the built-in configuration options, and can
-then be overwritten by any command line `--option` and `--compopt` arguments to
-generate the final application configuration.
-
-##### `--resources <filename>`
-
-Filename for the resources configuration. This
-will be merged with -and can overwrite- the built-in resource options, and can
-then be overwritten by any command line `--resopt`, `--rescompopt` and `--component`
-arguments to generate the final resource configuration.
-
-
-##### `--image path`
-
-The full path in Hadoop HDFS to a `.tar` or `.tar.gz` file containing
-the binaries needed to run the target application.
-
-Example
-
- --image hdfs://namenode/shared/binaries/hbase-0.96.tar.gz
-
-##### `--component <name> <count>`
-
-The desired number of instances of a component
-
-
-Example
-
- --component worker 16
-
-This just sets the `component.instances` value of the named component's resource configuration.
-it is exactly equivalent to
-
- --rco worker component.instances 16
-
-
-
-#### `--compopt <component> <option> <value>`
-
-Provide a specific application configuration option for a component
-
-Example
-
- --compopt master env.TIMEOUT 10000
-
-These options are saved into the `app_conf.json` file; they are not used to configure the YARN Resource
-allocations, which must use the `--rco` parameter
-
-#### Resource Component option `--rescompopt` `--rco`
-
-`--rescompopt <component> <option> <value>`
-
-Set any role-specific option, such as its YARN memory requirements.
-
-Example
-
- --rco worker master yarn.memory 2048
- --rco worker worker yarn.memory max
-
-
-##### `--zkhosts host1:port1,[host2:port2,host3:port3, ...] `
-
-The zookeeper quorum.
-
-Example
-
- --zkhosts zk1:2181,zk2:2181,zk3:4096
-
-If unset, the zookeeper quorum defined in the property `slider.zookeeper.quorum`
-is used
-
-### `destroy <name>`
-
-Destroy a (stopped) applicaton instance .
-
-Important: This deletes all persistent data
-
-Example
-
- slider destroy instance1
-
-### `exists <name> [--live]`
-
-Probe the existence of the named Slider application instance. If the `--live` flag is set, the instance
-must actually be running
-
-If not, an error code is returned.
-
-When the --live` flag is unset, the command looks for the application instance to be
-defined in the filesystem -its operation state is not checked.
-
-Return codes
-
- 0 : application instance is defined in the filesystem
- 70 : application instance is unknown
-
-Example:
-
- slider exists instance4
-
-#### Live Tests
-
-When the `--live` flag is set, the application instance must be running for the command
-to succeed
-
-1. The probe does not check the status of any Slider-deployed services, merely that a application instance has been deployed
-1. A application instance that is finished or failed is not considered to be live.
-
-Return codes
-
- 0 : application instance is running
- -1 : application instance exists but is not running
- 70 : application instance is unknown
-
-
-Example:
-
- slider exists instance4 --live
-
-### `flex <name> [--component component count]* `
-
-Flex the number of workers in an application instance to the new value. If greater than before, new copies of the component will be requested. If less, component instances will be destroyed.
-
-This operation has a return value of 0 if the size of a running instance was changed.
-
-It returns -1 if there is no running application instance, or the size of the flexed instance matches that of the original -in which case its state does not change.
-
-Example
-
- slider flex instance1 --component worker 8 --filesystem hdfs://host:port
- slider flex instance1 --component master 2 --filesystem hdfs://host:port
-
-
-### `freeze <name> [--force] [--wait time] [--message text]`
-
-freeze the application instance. The running application is stopped. Its settings are retained in HDFS.
-
-The `--wait` argument can specify a time in seconds to wait for the application instance to be frozen.
-
-The `--force` flag causes YARN asked directly to terminate the application instance.
-The `--message` argument supplies an optional text message to be used in
-the request: this will appear in the application's diagnostics in the YARN RM UI.
-
-If an unknown (or already frozen) application instance is named, no error is returned.
-
-Examples
-
- slider freeze instance1 --wait 30
- slider freeze instance2 --force --message "maintenance session"
-
-
-### `list <name>`
-
-List running Slider application instances visible to the user.
-
-If an instance name is given and there is no running instance with that name, an error is returned.
-
-Example
-
- slider list
- slider list instance1
-
-### `registry (--list | --listconf | --getconf <conf>) [--name <name>] [--servicetype <servicetype>] [--verbose]`
-
-List registered application instances visible to the user. This is slightly
-different from the `slider list` command in that it does not make use of the
-YARN application list. Instead it communicates with Zookeeper -and works
-with any applications which has registered itself with the "service registry"
-
-The `--name <name>` option names the registry entry to work with. For slider applications,
-this is the application instance
-
-
-The `--servicetype <servicetype>` option allows a different service type to be chosen.
-The default is `"org.apache.slider`
-
-The `--verbose` flag triggers more verbose output on the operations
-
-The `--internal` flag indicates the configurations to be listed and retrieved
-are from the "internal" list of configuration data provided for use within a
-deployed application.
-
-There are two common exit codes, the exact values being documented
-in [Exit Codes](exitcodes.md)
-
-1. If there is no matching service then the operation fails with the
-`EXIT_NOT_FOUND` code (77).
-2. If there are no configurations in a listing, or the named configuration
-is not found, the command returns the exit code `EXIT_NOT_FOUND` (77)
-
-Operations:
-
-#### `registry --list [--servicetype <servicetype>] [--name <name>] [--verbose]`
-
-List all services of the service type and optionally the name.
-
-
-
-#### `registry --listconf [--name <name>] [--internal] [--servicetype <servicetype>]`
-
-List the configurations exported by of a named application
-
-
-
-#### `registry --getconf <configuration> [--format (xml|json|properties)] [--dest <path>] [--internal] ` get the configuration
-
-Get a named configuration in a chosen format. Default: XML
-
- `--dest <path>` : the filename or directory to save a configuration to.
- `--format (xml|json|properties)` defines the output format
-
-
-
-
-
-
-
-### `status <name> [--out <filename>]`
-
-Get the status of the named application instance in JSON format. A filename can be used to
-specify the destination.
-
-Examples:
-
- slider status instance1 --manager host:port
-
- slider status instance2 --manager host:port --out status.json
-
-
-
-### `thaw <name> [--wait time`]
-
-Resume a frozen application instance, recreating it from its previously saved state. This will include a best-effort attempt to create the same number of nodes as before, though their locations may be different.
-
-Examples:
-
- slider thaw instance2
- slider thaw instance1 --wait 60
-
-
-If the application instance is already running, this command will not affect it.
-
-
-### `version`
-
-The command `slider version` prints out information about the compiled
-Slider application, the version of Hadoop against which it was built -and
-the version of Hadoop that is currently on its classpath.
-
-Note that this is the client-side Hadoop version, not that running on the server, though
-that can be obtained in the status operation
-
-
-
-## Commands for testing
-
-
-These are clearly abnormal operations; they are here primarily for testing
--and documented for completeness.
-
-### `kill-container <name> --id container-id`
-
-Kill a YARN container belong to the application. This is useful primarily for testing the
-resilience to failures.
-
-Container IDs can be determined from the application instance status JSON document.
-
-
-### `am-suicide <name> [--exitcode code] [--message message] [--wait time]`
-
-This operation is purely for testing Slider Application Master restart;
-it triggers an asynchronous self-destruct operation in the AM -an
-operation that does not make any attempt to cleanly shut down the process.
-
-If the application has not exceeded its restart limit (as set by
-`slider.yarn.restart.limit`), YARN will attempt to restart the failed application.
-
-Example
-
- slider am-suicide --exitcode 1 --wait 5000 -message "test"
-
-<!--- ======================================================================= -->
-
-
-## Instance Naming
-
-Application instance names must:
-
-1. be at least one character long
-1. begin with a lower case letter
-1. All other characters must be in the range \[a-z,0-9,_]
-1. All upper case characters are converted to lower case
-
-Example valid names:
-
- slider1
- storm4
- hbase_instance
- accumulo_m1_tserve4
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/a_YARN_service_registry.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/a_YARN_service_registry.md b/src/site/markdown/registry/a_YARN_service_registry.md
deleted file mode 100644
index b695106..0000000
--- a/src/site/markdown/registry/a_YARN_service_registry.md
+++ /dev/null
@@ -1,226 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# A YARN Service Registry for Apache Slider
-
-## April 2014
-
-# Introduction
-
-This document looks at the needs and options of a service registry.
-
-The core issue is that as the location(s) of a dynamically deployed application are unknown, the standard Hadoop and Java configuration model of some form of text files containing hostnames, ports and URLS no longer works. You cannot define up-front where a service will be.
-
-Some Hadoop applications -HBase and Accumulo -have solved this with custom ZK bindings. This works for the specific clients, but requires hbase and accumulo client JARs in order to be able to work with the content. (or a re-implementation with knowledge of the non-standard contents of the ZK nodes)
-
-Other YARN applications will need to publish their bindings - this includes, but is not limited to- Slider deployed applications. Again, these applications can use their own registration and binding model, which would again require custom clients to locate the registry information and parse the contents.
-
-YARN provides some minimal publishing of AM remote endpoints: a URL to what is assumed to be a Web UI (not a REST API), and an IPC port. The URL is displayed in the YARN UI -in which case it is accessed via a proxy which (currently) only support HTTP GET operations. The YARN API call to list all applications can be used to locate a named instance of an application by (user, application-type, name), and then obtain the raw URL and IPC endpoints. This enumeration process is an O(apps) operation on the YARN RM and only provides access to those two endpoints. Even with the RAW URL, REST operations have proven "troublesome", due to a web filter which redirects all direct requests to the proxy -unless it comes from the same host as the proxy.
-
-Hadoop client applications tend to retrieve all their configuration information from files in the local filesystem, hadoop-site.xml, hdfs-site.xml, hbase-site.xml, etc. This requires the configuration files to be present on all systems. Tools such as Ambari can keep the files in the server up to date -assuming a low rate of change- ---but these tools do nothing for the client applications themselves. It is up to the cluster clients to (somehow) retrieve these files, and to keep their copies up to date. *This is a problem that exists with today's applications*.
-
-As an example, if a YARN client does not know the value of "yarn.application.classpath", it cannot successfully deploy any application in the YARN cluster which needs the cluster-side Hadoop and YARN JARs on its application master's classpath. This is not a theoretical problem, as some clusters have a different classpath from the default: without a correct value the Slider AM does not start. And, as it is designed to be run remotely, it cannot rely on a local installation of YARN to provide the correct values ([YARN-973](https://issues.apache.org/jira/browse/YARN-973)).
-
-# What do we need?
-
-**Discovery**: An IPC and URL discovery system for service-aware applications to use to look up a service to which it wishes to talk to. This is not an ORB -it's not doing redirection -, but it is something that needs to be used before starting IPC or REST communications.
-
-**Configuration**: A way for clients of a service to retrieve more configuration data than simply the service endpoints. For example: everything needed to create a site.xml document.
-
-## Client-side
-
-* Allow clients of a YARN application to locate the service instance and its service ports (web, IPC, REST...) efficiently even on a large YARN cluster.
-
-* Allow clients to retrieve configuration values which can be processed client-side into the configuration files and options which the application needs
-
-* Give clients confidence that the service with which they interact is the one they expect to interact with -not another potentially malicious service deployed by a different user.
-
-* clients to be able to watch a service and retrieve notification of changes
-
-* cross-language support.
-
-## For all Services
-
-* Allow services to publish their binding details for the AM and of code running in the containers (which may be published by the containers)
-
-* Use entries in registry as a way of enforcing uniqueness of the instance (app, owner, name)?
-
-* values to update when a service is restarted on a different host
-
-* values to indicate when a service is not running. This may be implicit "no entry found" or explicit "service exists but not running"
-
-* Services to be able to act as clients to other services
-
-## For Slider Services (and presumably others)
-
-* Ability to publish information about configuration documents that can be retrieved -and URLs
-
-* Ability to publish facts internal to the application (e.g. agent reporting URLs)
-
-* Ability to use service paths as a way to ensure a single instance of a named service can be deployed by a user
-
-## Management and PaaS UIs
-
-* Retrieve lists of web UI URLs of AM and of deployed components
-
-* Enum components and their status
-
-* retrieve dynamic assignments of IPC ports
-
-* retrieve dynamic assignments of JMX ports
-
-* retrieve any health URLs for regular probes
-
-* Listen to changes in the service mix -the arrival and departure of service instances, as well as changes in their contents.
-
-
-
-## Other Needs
-
-* Registry-configured applications. In-cluster applications should be able to subscribe to part of the registry
-to pick up changes that affect them -both for their own application configuration, and for details about
-applications on which they depend themselves.
-
-* Knox: get URLs that need to be converted into remote paths
-
-* Cloud-based deployments: work on virtual infrastructures where hostnames are unpredictable.
-
-# Open Source Registry code
-
-What can we use to implement this from ASF and ASF-compatible code?
-
-## Zookeeper
-
-We'd need a good reason not to use this. There are still some issues
-
-1. Limits on amount of published data?
-
-2. Load limits, especially during cluster startup, or if a 500-mapper job all wants to do a lookup.
-
-3. Security story
-
-4. Impact of other ZK load on the behaviour of the service registry -will it cause problems if overloaded -and are they recoverable?
-
-## Apache Curator
-
-Netflix's core curator -now [Apache Curator](http://curator.apache.org/)- framework adds a lot to make working with ZK easier, including pluggable retry policies, binding tools and other things.
-
-There is also its "experimental" [service discovery framework](http://curator.apache.org/curator-x-discovery-server/index.html), which
-
-1. Allows a service to register a URL with a name and unique ID (and custom metadata). multiple services of a given name can be registered
-
-2. Allows a service to register >1 URL.
-
-3. Has a service client which performs lookup and can cache results.
-
-4. Has a REST API
-
-Limitations
-
-* The service discovery web UI and client does not work with the version of
-Jackson (1.8.8) in Hadoop 2.4. The upgraded version in Hadoop 2.5 is compatible [HADOOP-10104](https://issues.apache.org/jira/browse/HADOOP-10104).
-
-* The per-entry configuration payload attempts to get jason to perform Object/JSON mapping with the classname provided as an attribute in the JSON. This destroys all ability of arbitrary applications to parse the published data, as well as cross-language clients -is brittle and morally wrong from a data-sharing perspective.
-
- {
-
- "name" : "name",
- "id" : "service",
- "address" : "localhost",
- "port" : 8080,
- "sslPort" : 443,
- "payload" : {
- "@class" : "org.apache.slider.core.registry.ServiceInstanceData",
- "externalView" : {
- "key" : "value"
- }
- },
- "registrationTimeUTC" : 1397249829062,
- "serviceType" : "DYNAMIC",
- "uriSpec" : {
- "parts" : [ {
- "value" : "http:",
- "variable" : false
- }, {
- "value" : ":",
- "variable" : false
- } ]
- }
- }
-
-
-
-## [Helix Service Registry](http://helix.apache.org/0.7.0-incubating-docs/recipes/service_discovery.html)
-
-This is inside Helix somewhere, used in LI in production at scale -and worth looking at. LI separate their Helix Zookeeper Quorum from their application-layer quorum, to isolate load.
-
-Notable features
-
-1. The registry is also the liveness view of the deployed application. Client's aren't watching the service registry for changes, they are watching Helix's model of the deployed application.
-1. The deployed application can pick up changes to its state the same way, allowing for live application manipulation.
-1. Tracks nodes that continually join/leave the group and drops them as unreliable.
-
-## Twill Service Registry
-
-Twill's [service registry code](http://twill.incubator.apache.org/apidocs/index.html), lets applications register a [(hostname, port)](http://twill.incubator.apache.org/apidocs/org/apache/twill/discovery/Discoverable.html) pair in the registry by a name, a name by which clients can look up and enumerate all services with a specific name.
-
-Clients can subscribe to changes in the list of services with a specific name -so picking up the arrival and departure of instances, and probe to see if a previously discovered entity is still registered.
-
-Zookeeper- and in-memory registry implementations are provided.
-
-One nice feature about this architecture -and Twill in general- is that its general single-method callback model means that it segues nicely into Java-8 lambda-expressions. This is something to retain in a YARN-wide service registry.
-
-Comparing it to curator, it offers a proper subset of curator's registered services [ServiceInstance](http://curator.apache.org/apidocs/org/apache/curator/x/discovery/ServiceInstance.html) -implying that you could publish and retrieve Curator-registered services via a new implementation of Twill's DiscoveryService. This would require extensions to the curator service discovery client allow ZK nodes to be watched for changes. This is a feature that would be useful in many use cases -such as watching service availability across a cluster, or simply blocking until a dependent service was launched.
-
-As with curator, the amount of information that can be published isn't enough for management tools to make effective use of the service registration, while for slider there's no way to publish configuration data. However a YARN registry will inevitably be a superset of the Twill client's enumerated and retrieved data -so if its registration API were sufficient to register a minimal service, supporting the YARN registry via Twill's existing API should be straightforward.
-
-## Twitter Commons Service Registration
-
-[Twitter Commons](https://github.com/twitter/commons) has a service registration library, which allows for registration of sets of servers, [publishing the hostname and port of each](http://twitter.github.io/commons/apidocs/com/twitter/common/service/registration/package-tree.html)., along with a map of string properties.
-
-Zookeeper based, it suffices if all servers are identical and only publishing single (hostname, port) pairs for callers.
-
-## AirBnB Smartstack
-
-SmartStack is [Air BnB's cloud-based service discovery system](http://nerds.airbnb.com/smartstack-service-discovery-cloud/).
-
-It has two parts, *Nerve* and *Synapse*:
-
-[**Nerve**](https://github.com/airbnb/nerve) is a ruby agent designed to monitor processes and register healthy instances in ZK (or to a mock reporter). It includes [probes for TCP ports, HTTP and rabbitMQ](https://github.com/airbnb/nerve/tree/master/lib/nerve/service_watcher). It's [a fairly simple liveness monitor](https://github.com/airbnb/nerve/blob/master/lib/nerve/service_watcher.rb).
-
-[**Synapse**](https://github.com/airbnb/synapse) takes the data and uses it to configure [HAProxy instances](http://haproxy.1wt.eu/). HAProxy handles the load balancing, queuing and integrating liveness probes into the queues. Synapse generates all the configuration files for an instance -but also tries to reconfigure the live instances via their socket APIs,
-
-Alongside these, AirBnB have another published project on Github, [Optica](https://github.com/airbnb/optica), which is a web application for nodes to register themselves with (POST) and for others to query. It publishes events to RabbitMQ, and again uses ZK to store state.
-
-AirBnB do complain a bit about ZK and its brittleness. They do mention that they suspect it is due to bugs in the Ruby ZK client library. This may be exacerbated by in-cloud deployments. Hard-coding the list of ZK nodes may work for a physical cluster, but in a virtualized cluster, the hostnames/IP Addresses of those nodes may change -leading to a meta-discovery problem: how to find the ZK quorum -especially if you can't control the DNS servers.
-
-## [Apache Directory](http://directory.apache.org/apacheds/)
-
-This is an embeddable LDAP server
-
-* Embeddable inside Java apps
-
-* Supports Kerberos alongside X.500 auth. It can actually act as a Key server and TGT if desired.
-
-* Supports DNS and DHCP queries.
-
-* Accessible via classic LDAP APIs.
-
-This isn't a registry service directly, though LDAP queries do make enumeration of services *and configuration data* straightforward. As LDAP libraries are common across languages -even built in to the Java runtime- LDAP support makes publishing information to arbitrary clients relatively straightforward.
-
-If service information were to be published via LDAP, then it should allow IT-managed LDAP services to both host this information, and publish configuration data. This would be relevant for classic Hadoop applications if we were to move the Configuration class to support back-end configuration sources beyond XML files on the classpath.
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/index.md b/src/site/markdown/registry/index.md
deleted file mode 100644
index 148b4bc..0000000
--- a/src/site/markdown/registry/index.md
+++ /dev/null
@@ -1,47 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Service Registry
-
-The service registry is a core part of the Slider Architecture -it is how
-dynamically generated configurations are published for clients to pick up.
-
-The need for a service registry goes beyond Slider, however. We effectively
-have application-specific registries for HBase and Accumulo, and explicit
-registries in Apache Helix and Apache Twill, as well as re-usable registry
-code in Apache Curator.
-
-[YARN-913](https://issues.apache.org/jira/browse/YARN-913) covers the need
-for YARN itself to have a service registry. This would be the ideal ultimate
-solution -it would operate at a fixed location/ZK path, and would be guaranteed
-to be on all YARN clusters, so code could be written expecting it to be there.
-
-It could also be used to publish binding data from static applications,
-including HBase, Accumulo, Oozie, -applications deployed by management tools.
-Unless/until these applications self-published their binding data, it would
-be the duty of the management tools to do the registration.
-
-
-
-## Contents
-
-1. [YARN Application Registration and Binding: the Problem](the_YARN_application_registration_and_binding_problem.html)
-1. [A YARN Service Registry](a_YARN_service_registry.html)
-1. [April 2014 Initial Registry Design](initial_registry_design.html)
-1. [Service Registry End-to-End Scenarios](service_registry_end_to_end_scenario.html)
-1. [P2P Service Registries](p2p_service_registries.html)
-1. [References](references.html)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/initial_registry_design.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/initial_registry_design.md b/src/site/markdown/registry/initial_registry_design.md
deleted file mode 100644
index e132b39..0000000
--- a/src/site/markdown/registry/initial_registry_design.md
+++ /dev/null
@@ -1,110 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# April 2014 Initial Registry Design for Apache Slider
-
-This is the plan for the initial registry design.
-
-1. Use Apache Curator [service discovery code](http://curator.apache.org/curator-x-discovery/index.html).
-
-2. AMs to register as (user, name). Maybe "service type" if we add that as an option in the slider configs
-
-3. Lift "external view" term from Helix -concept that this is the public view, not internal.
-
-4. application/properties section to list app-wide values
-
-5. application/services section to list public service URLs; publish each as unique-ID-> (human name, URL, human text). code can resolve from UniqueID; UIs can use human data.
-
-6. String Template 2 templates for generation of output (rationale: library for Python Java, .NET)
-
-7. Java CLI to retrieve values from ZK and apply named template (local, hdfs). Include ability to restrict to list of named properties (pattern match).
-
-8. AM to serve up curator service (later -host in RM? elsewhere?)
-
-### forwards-compatilibity
-
-1. This initial design will hide the fact that Apache Curator is being used to discover services,
-by storing information in the payload, `ServiceInstanceData` rather than in (the minimdal) curator
-service entries themselves. If we move to an alternate registry, provided we
-can use the same datatype -or map to it- changes should not be visible.
-
-1. The first implementation will not support watching for changes.
-
-### Initial templates
-
-* hadoop XML conf files
-
-* Java properties file
-
-* HTML listing of services
-
-
-
-## Example Curator Service Entry
-
-This is the prototype's content
-
-Toplevel
-
- service CuratorServiceInstance{name='slider', id='stevel.test_registry_am', address='192.168.1.101', port=62552, sslPort=null, payload=org.apache.slider.core.registry.info.ServiceInstanceData@4e9af21b, registrationTimeUTC=1397574073203, serviceType=DYNAMIC, uriSpec=org.apache.curator.x.discovery.UriSpec@ef8dacf0}
-
-Slider payload.
-
- payload=
- {
- "internalView" : {
- "endpoints" : {
- "/agents" : {
- "value" : "http://stevel-8.local:62552/ws/v1/slider/agents",
- "protocol" : "http",
- "type" : "url",
- "description" : "Agent API"
- }
- },
- "settings" : { }
- },
-
- "externalView" : {
- "endpoints" : {
- "/mgmt" : {
- "value" : "http://stevel-8.local:62552/ws/v1/slider/mgmt",
- "protocol" : "http",
- "type" : "url",
- "description" : "Management API"
- },
-
- "slider/IPC" : {
- "value" : "stevel-8.local/192.168.1.101:62550",
- "protocol" : "org.apache.hadoop.ipc.Protobuf",
- "type" : "address",
- "description" : "Slider AM RPC"
- },
- "registry" : {
- "value" : "http://stevel-8.local:62552/ws/registry",
- "protocol" : "http",
- "type" : "url",
- "description" : "Registry"
- }
- },
- "settings" : { }
- }
- }
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/p2p_service_registries.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/p2p_service_registries.md b/src/site/markdown/registry/p2p_service_registries.md
deleted file mode 100644
index eaf5097..0000000
--- a/src/site/markdown/registry/p2p_service_registries.md
+++ /dev/null
@@ -1,137 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# P2P Service Registries for Apache Slider
-
-Alongside the centralized service registries, there's much prior work on
-P2P discovery systems, especially for mobile and consumer devices.
-
-They perform some multicast- or distributed hash table-based lookup,
-and tend to have common limitations:
-
-* scalability
-
-* the bootstrapping problem
-
-* security: can you trust the results to be honest?
-
-* consistency: can you trust the results to be complete and current?
-
-Bootstrapping is usually done via multicast, possibly then switching
-to unicast for better scale. As multicasting doesn't work in cloud
-infrastructures, none of the services work unmodified in public
-clouds. There's multiple anecdotes of
-[Amazon's SimpleDB service](http://aws.amazon.com/simpledb/) being used as a
-registry for in-EC2 applications. At the very least, this service and its
-equivalents in other cloud providers could be used to bootstrap ZK client
-bindings in cloud environments.
-
-## Service Location Protocol
-
-Service Location Protocol is a protocol for discovery services that came out
-of Sun, Novell and others -it is still available for printer discovery and
-suchlike
-
-It supports both a multicast discovery mechanism, and a unicast protocol
-to talk to a Directory Agent -an agent that is itself discovered by multicast
-requests, or by listening for the agent's intermittent multicast announcements.
-
-There's an extension to DHCP, RFC2610, which added the ability for DHCP to
-advertise Directory Agents -this was designed to solve the bootstrap problem
-(though not necessarily security or in-cloud deployment). Apart from a few
-mentions in Windows Server technical notes, it does not appear to exist.
-
-* [[RFC2608](http://www.ietf.org/rfc/rfc2608.txt)] *Service Location Protocol, Version 2* , IEEE, 1999
-
-* [[RFC3224](http://www.ietf.org/rfc/rfc3224.txt)] *Vendor Extensions for Service Location Protocol, Version 2*, IETF, 2003
-
-* [[RFC2610](http://www.ietf.org/rfc/rfc2610.txt)] *DHCP Options for Service Location Protocol, IETF, 1999*
-
-## [Zeroconf](http://www.zeroconf.org/)
-
-The multicast discovery service implemented in Apple's Bonjour system
---multicasting DNS lookups to all peers in the subnet.
-
-This allows for URLs and hostnames to be dynamically positioned, with
-DNS domain searches allowing for enumeration of service groups.
-
-This protocol scales very badly; the load on *every* client in the
-subnet is is O(DNS-queries-across-subnet), hence implicitly `O(devices)*O(device-activity)`.
-
-The special domains `_tcp.`, `_udp.` and their subdomains can also be
-served up via a normal DNS server.
-
-## [Jini/Apache River](http://river.apache.org/doc/specs/html/lookup-spec.html)
-
-Attribute-driven service enumeration, which drives the, Java-client-only
-model of downloading client-side code. There's no requirement for the remote
-services to be in Java, only that drivers are.
-
-## [Serf](http://www.serfdom.io/)
-
-This is a library that implements the [SWIM protocol](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf) to propagate information around a cluster. Apparently works in virtualized clusters too. It's already been used in a Flume-on-Hoya provider.
-
-## [Anubis](http://sourceforge.net/p/smartfrog/svn/HEAD/tree/trunk/core/components/anubis/)
-
-An HP Labs-built [High Availability tuple-space](http://sourceforge.net/p/smartfrog/svn/HEAD/tree/trunk/core/components/anubis/doc/HPL-2005-72.pdf?format=raw) in SmartFrog; used in production in some of HP's telco products. An agent publishes facts into the T-Space, and within one heartbeat all other agents have it. One heart-beat later, unless there's been a change in the membership, the publisher knows the others have it. One heartbeat later the agents know the publisher knows it, etc.
-
-Strengths:
-
-* The shared knowledge mechanism permits reasoning and mathematical proofs.
-
-* Strict ordering between heartbeats implies an ordering in receipt.
-This is stronger than ZK's guarantees.
-
-* Lets you share a moderate amount of data (the longer the heartbeat
-interval, the more data you can publish).
-
-* Provided the JVM hosting the Anubis agent is also hosting the service,
-liveness is implicit
-
-* Secure to the extent that it can be locked down to allow only nodes with
-mutual trust of HTTPS certificates to join the tuple-space.
-
-Weaknesses
-
-* (Currently) bootstraps via multicast discovery.
-
-* Brittle to timing, especially on virtualized clusters where clocks are unpredictable.
-
-It proved good for workload sharing -tasks can be published to it, any
-agent can say "I'm working on it" and take up the work. If the process
-fails, the task becomes available again. We used this for distributed scheduling in a rendering farm.
-
-## [Carmen](http://www.hpl.hp.com/techreports/2002/HPL-2002-257)
-
-This was another HP Labs project, related to the Cooltown "ubiquitous
-computing" work, which was a decade too early to be relevant. It was
-also positioned by management as a B2B platform, so ended up competing
-with - and losing against - WS-* and UDDI.
-
-Carmen aimed to provide service discovery with both fixed services, and
-with highly mobile client services that will roam around the network -they
-are assumed to be wireless devices.
-
-Services were published with and searched for by attributed, locality
-was considered to be a key attribute -local instances of a service
-prioritized. Those services with a static location and low rate of
-change became the stable caches of service information -becoming,
-as with skype, "supernodes".
-
-Bootstrapping the cluster relied on multicast, though alternatives
-based on DHCP and DNS were proposed.
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/references.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/references.md b/src/site/markdown/registry/references.md
deleted file mode 100644
index a5afb43..0000000
--- a/src/site/markdown/registry/references.md
+++ /dev/null
@@ -1,49 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Service Registry References
-
-Service registration and discovery is a problem in distributed computing that has been explored for over thirty years, with
-[Birrell81]'s *Grapevine* system the first known implementation -though of
-
-# Papers
-
-* **[Birrell81]** Birrell, A. et al, [*Grapevine: An exercise in distributed computing*](http://research.microsoft.com/apps/pubs/default.aspx?id=63661). Comm. ACM 25, 4 (Apr 1982), pp260-274.
-The first documented directory service; relied on service shutdown to resolve update operations.
-
-* **[Das02]** [*SWIM: Scalable Weakly-consistent Infection-style Process Group Membership Protocol*](http://www.cs.cornell.edu/~asdas/research/dsn02-swim.pdf)
-P2P gossip-style data sharing protocol with random liveness probes to address scalable liveness checking. Ceph uses similar liveness checking.
-
-* **[Marti02]** Marti S. and Krishnam V., [*Carmen: A Dynamic Service Discovery Architecture*](http://www.hpl.hp.com/techreports/2002/HPL-2002-257),
-
-* **[Lampson86]** Lampson, B. [*Designing a Global Naming Service*](http://research.microsoft.com/en-us/um/people/blampson/36-GlobalNames/Acrobat.pdf). DEC.
-Distributed; includes an update protocol and the ability to add links to other parts of the tree. Also refers to [*Xerox Clearinghouse*](http://bitsavers.informatik.uni-stuttgart.de/pdf/xerox/parc/techReports/OPD-T8103_The_Clearinghouse.pdf), which apparently shipped.
-
-* **[Mockapetris88]** Mockapetris, P. [*Development of the domain name system*](http://bnrg.eecs.berkeley.edu/~randy/Courses/CS268.F08/papers/31_dns.pdf). The history of DNS
-
-* **[Schroeder84]** Schroeder, M.D. et al, [*Experience with Grapevine: The Growth of a Distributed System*](http://research.microsoft.com/apps/pubs/default.aspx?id=61509). Xerox.
-Writeup of the experiences of using grapevine, with its eventual consistency and lack of idempotent message delivery called out -along with coverage of operations issues.
-
-* **[van Renesse08]** van Renesse, R. et al, [*Astrolabe: A Robust and Scalable Technology For Distributed System Monitoring, Management, and Data Mining*](http://www.cs.cornell.edu/home/rvr/papers/astrolabe.pdf). ACM Transactions on Computer Systems
-Grandest P2P management framework to date; the work that earned Werner Vogel his CTO position at Amazon.
-
-* **[van Steen86]** van Steen, M. et al, [*A Scalable Location Service for Distributed Objects*](http://www.cs.vu.nl/~ast/publications/asci-1996a.pdf).
-Vrije Universiteit, Amsterdam. Probably the first Object Request Broker
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/registry-model.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/registry-model.md b/src/site/markdown/registry/registry-model.md
deleted file mode 100644
index 5ba2a5b..0000000
--- a/src/site/markdown/registry/registry-model.md
+++ /dev/null
@@ -1,75 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: Service Registry
-
-The service registry model is designed to support dynamically
-deployed Slider applications, *and* statically deployed versions
-of the same application -provided the latter also registers itself,
-its public network services, and any configurations and files
-that it wishes clients to be able to retrieve.
-
-The architecture and implementation of this registry is not defined
-here -instead the view of it seen by clients.
-
-1. A 'service registry' exists in the YARN cluster into which
-services can be registered.
-
-1. There is no restriction on the number of services that can be registered in
-the registry, the type of service that may register, or even on how many
-registered services an application running in the YARN cluster may register.
-
-1. Services are registered by their type, owner and name. As an example,
-Alice's slider-managed HBase cluster `ingress` would have a type `org.apache.hbase`,
-owner `alice` and name `ingress`.
-
-1. In the case of Slider-managed services, there is a separate slider instance
-registration which publishes information about slider itself. In the example
-above, this would be (`org.apache.slider`,`alice`,`ingress`).
-
-1. Services can publish information about themselves, with common entries being:
-
- * service name and description.
- * URLs of published web UIs and web service APIs
- * network address of other protocols
-
-1. Services may also publish.
-
- * URLs to configuration details
- * URLs documents published for use as client-side configuration data -either
- directly or through some form of processing.
- * public service-specific data, for use by applications that are aware of
- the specific service type.
- * internal service-specific data -for use by the components that comprise
- an application. This allows the registry to be used to glue together
- the application itself.
-
-1. Services can be listed and examined.
-
-1. Service-published configuration key-value pairs can be retrieved by clients
-
-1. Service-published documents (and packages of such documents) can be
-retrieved by clients.
-
-1. There's no requirement for service instances to support any standard protocols;
-
-1. Some protocols are defined which they MAY implement. For example, the protocol
-to enumerate and retrieve configuration documents is designed to be implemented
-by any service that wishes to publish such content.
-
-1. In a secure cluster, the registry data may be restricted, along with any
-service protocols offered by the registered services.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/service_registry_end_to_end_scenario.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/service_registry_end_to_end_scenario.md b/src/site/markdown/registry/service_registry_end_to_end_scenario.md
deleted file mode 100644
index 4d15fa4..0000000
--- a/src/site/markdown/registry/service_registry_end_to_end_scenario.md
+++ /dev/null
@@ -1,156 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: Service Registry End-to-End Scenarios
-
-## AM startup
-
-1. AM starts, reads in configuration, creates provider
-
-2. AM builds web site, involving provider in process (*there's a possible race condition here, due to the AM registration sequence)*
-
-3. AM registers self with RM, including web and IPC ports, and receives list of existing containers; container loss notifications come in asynchronously *(which is why the AM startup process is in a synchronized block)*
-
-4. AM inits it's `ApplicationState` instance with the config, instance description and RM-supplied container list.
-
-5. AM creates service registry client using ZK quorum and path provided when AM was started
-
-6. AM registers standard endpoints: RPC, WebUI, REST APIs
-
-7. AM registers standard content it can serve (e.g `yarn-site.xml`)
-
-8. AM passes registry to provider in `bind()` operation.
-
-9. AM triggers review of application state, requesting/releasing nodes as appropriate
-
-## Agent Startup: standalone
-
-1. Container is issued to AM
-
-2. AM chooses component, launches agent on it -with URL of AM a parameter (TODO: Add registry bonding of ZK quorum and path)
-
-3. Agent starts up.
-
-4. Agent locates AM via URL/ZK info
-
-5. Agent heartbeats in with state
-
-6. AM gives agent next state command.
-
-## AM gets state from agent:
-
-1. Agent heartbeats in
-
-2. AM decides if it wants to receive config
-
-3. AM issues request for state information -all (dynamic) config data
-
-4. Agent receives it
-
-5. Agent returns all config state, including: hostnames, allocated ports, generated values (e.g. database connection strings, URLs) - as two-level (allows agent to define which config options are relevant to which document)
-
-## AM saves state for serving
-
-1. AM saves state in RAM (assumptions: small, will rebuild on restart)
-
-2. AM updates service registry with list of content that can be served up and URLs to retrieve them.
-
-3. AM fields HTTP GET requests on content
-
-## AM Serves content
-
-A simple REST service serves up content on paths published to the service registry. It is also possible to enumerate documents published by GET operations on parent paths.
-
-1. On GET command, AM locates referenced agent values
-
-2. AM builds up response document from K-V pairs. This can be in a limited set of formats: Hadoop XML, Java properties, YAML, CSV, HTTP, JSON chosen as ? type param. (this generation is done from template processing in AM using slider.core.template module)
-
-3. response is streamed with headers of : `content-type`, `content-length`, do not cache in proxy, expires,* (with expiry date chosen as ??)*
-
-# Slider Client
-
-Currently slider client enumerates the YARN registry looking for slider instances -including any instances of the same application running before launching a cluster.
-
-This
-
-* has race conditions
-* has scale limitations `O(apps-in-YARN-cluster)` + `O(completed-apps-in-RM-memory)`
-* only retrieves configuration information from slider-deployed application instances. *We do not need to restrict ourselves here.*
-
-## Slider Client lists applications
-
- slider registry --list [--servicetype <application-type>]
-
-1. Client starts
-
-2. Client creates creates service registry client using ZK quorum and path provided in client config properties (slider-client.xml)
-
-3. Client enumerates registered services and lists them
-
-## Slider Client lists content published by an application instance
-
- slider registry <instance> --listconf [--servicetype <application-type>]
-
-1. Client starts
-
-2. Client creates creates service registry client using ZK quorum and path provided in client config properties (slider-client.xml)
-
-3. Client locates registered service entry -or fails
-
-4. Client retrieves service data, specifically the listing of published documents
-
-5. Client displays list of content
-
-## Slider Client retrieves content published by an application instance
-
- slider registry <instance> --getconf <document> [--format (xml|properties|text|html|csv|yaml|json,...) [--dest <file>] [--servicetype <application-type>]
-
-1. Client starts
-
-2. Client creates creates service registry client using ZK quorum and path provided in client config properties (slider-client.xml)
-
-3. Client locates registered service entry -or fails
-
-4. Client retrieves service data, specifically the listing of published documents
-
-5. Client locates URL of content
-
-6. Client builds GET request including format
-
-7. Client executes command, follows redirects, validates content length against supplied data.
-
-8. Client prints response to console or saves to output file. This is the path specified as a destination, or, if that path refers to a directory, to
-a file underneath.
-
-## Slider Client retrieves content set published by an application instance
-
-Here a set of documents published is retrieved in the desired format of an application.
-
-## Slider Client retrieves document and applies template to it
-
-Here a set of documents published is retrieved in the desired format of an application.
-
- slider registry <instance> --source <document> [--template <path-to-template>] [--outfile <file>] [--servicetype <application-type>]
-
-1. document is retrieved as before, using a simple format such as json to retrieve it.
-
-2. The document is parsed and converted back into K-V pairs
-
-3. A template using a common/defined template library is applied to the content , generating the final output.
-
-Template paths may include local filesystem paths or (somehow) something in a package file
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/registry/the_YARN_application_registration_and_binding_problem.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/registry/the_YARN_application_registration_and_binding_problem.md b/src/site/markdown/registry/the_YARN_application_registration_and_binding_problem.md
deleted file mode 100644
index ab2493d..0000000
--- a/src/site/markdown/registry/the_YARN_application_registration_and_binding_problem.md
+++ /dev/null
@@ -1,192 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider YARN Application Registration and Binding: the Problem
-
-## March 2014
-
-# How to bind client applications to services dynamically placed applications?
-
-
-There are some constraints here
-
-1. The clients may be running outside the cluster -potentially over long-haul links.
-
-1. The location of an application deployed in a YARN cluster cannot be predicted.
-
-1. The ports used for application service endpoints cannot be hard-coded
-or predicted. (Alternatively: if they are hard-coded, then Socket-In-Use exceptions may occur)
-
-1: As components fail and get re-instantiated, their location may change.
-The rate of this depends on cluster and application stability; the longer
- lived the application, the more common it is.
-
-Existing Hadoop client apps have a configuration problem of their own:
-how are the settings in files such as `yarn-site.xml`picked up by today's
-applications? This is an issue which has historically been out of scope
-for Hadoop clusters -but if we are looking at registration and binding
-of YARN applications, there should be no reason why
-static applications cannot be discovered and bonded to using the same mechanisms.
-
-# Other constraints:
-
-1. Reduce the amount of change needed in existing applications to a minimum
----ideally none, though some pre-launch setup may be acceptable.
-
-2. Prevent malicious applications from registering a service endpoints.
-
-3. Scale with #of applications and #of clients; not overload on a cluster partitioning.
-
-4. Offer a design that works with apps that are deployed in a YARN custer
-outside of Slider. Rationale: want a mechanism that works with pure-YARN apps
-
-## Possible Solutions:
-
-### ZK
-
-Client applications use ZK to find services (addresses #1, #2 and #3).
-Requires location code in the client.
-
-HBase and Accumulo do this as part of a failover-ready design.
-
-### DNS
-
-Client apps use DNS to find services, with custom DNS server for a
-subdomain representing YARN services. Addresses #1; with a shortened TTL and
-no DNS address caching, #3. #2 addressed only if other DNS entries are used to
- publish service entries.
-
-Should support existing applications, with a configuration that is stable
-over time. It does require the clients to not cache DNS addresses forever
-(this must be explicitly set on Java applications,
-irrespective of the published TTL). It generates a load on the DNS servers
-that is `O(clients/TTL)`
-
-Google Chubby offers a DNS service to handle this. ZK does not -yet.
-
-### Floating IP Addresses
-
-If the clients know/cache IP addresses of services, these addresses could be
-floated across service instances. Linux HA has floating IP address support,
-while Docker containers can make use of them, especially if an integrated DHCP
-server handles the assignment of IP addresses to specific containers.
-
-ARP caching is the inevitable problem here, but it is still less brittle than
-relying on applications to know not to cache IP addresses -and nor does it
-place so much on DNS servers as short-TTL DNS entries.
-
-### LDAP
-
-Enterprise Directory services are used to publish/locate services. Requires
-lookup into the directory on binding (#1, #2), re-lookup on failure (#3).
-LDAP permissions can prevent untrusted applications registering.
-
-* Works well with Windows registries.
-
-* Less common Java-side, though possible -and implemented in the core Java
-libraries. Spring-LDAP is focused on connection to an LDAP server
--not LDAP-driven application config.
-
-### Registration Web Service
-
- Custom web service registration services used.
-
-* The sole WS-* one, UDDI, does not have a REST equivalent
---DNS is assumed to take on that role.
-
-* Requires new client-side code anyway.
-
-### Zookeeper URL Schema
-
-Offer our own `zk://` URL; java & .NET implementations (others?) to resolve, browser plugins.
-
-* Would address requirements #1 & #3
-
-* Cost: non-standard; needs an extension for every application/platform, and
-will not work with tools such as CURL or web browsers
-
-### AM-side config generation
-
-App-side config generation-YARN applications to generate client-side
-configuration files for launch-time information (#1, #2).
-The AM can dynamically create these, and as the storage load is all in
-the AM, does not consume as much resources in a central server as would
-publishing it all to that central server.
-
-* Requires application to know of client-side applications to support -
-and be able to generate to their configuration information (i.e. formatted files).
-
-* Requires the AM to get all information from deployed application components
-needed to generate bindings. Unless the AM can resolve YARN App templates,
-need a way to get one of the components in the app to generate settings for
-the entire cluster, and push them back.
-
-* Needs to be repeated for all YARN apps, however deployed.
-
-* Needs something similar for statically deployed applications.
-
-
-### Client-side config generation
-
-YARN app to publish attributes as key-val pairs, client-side code to read and
-generate configs from (#1, #2). Example configuration generators could
-create: Hadoop-client XML, Spring, tomcat, guice configs, something for .NET.
-
-* Not limited to Hoya application deployments only.
-
-* K-V pairs need to be published "somewhere". A structured section in the
-ZK tree per app is the obvious location -though potentially expensive. An
-alternative is AM-published data.
-
-* Needs client-side code capable of extracting information from YARN cluster
-to generate client-specific configuration.
-
-* Assumes (key, value) pairs sufficient for client config generation. Again,
-some template expansion will aid here (this time: client-side interpretation).
-
-* Client config generators need to find and bind to the target application themselves.
-
-
-
-Multiple options:
-
-* Standard ZK structure for YARN applications (maybe: YARN itself to register
-paths in ZK and set up child permissions,so enforcing security).
-
-* Agents to push to ZK dynamic information as K-V pairs
-
-* Agent provider on AM to fetch K-V pairs and include in status requests
-
-* CLI to fetch app config keys, echo out responses (needs client log4j settings
-to log all slf/log4j to stderr, so that app > results.txt would save results explicitly
-
-* client side code per app to generate specific binding information
-
-### Load-balancer app Yarn App
-
-Spread requests round a set of registered handlers, e.g web servers. Support
-plugins for session binding/sharding.
-
-Some web servers can do this already; a custom YARN app could use grizzy
-embedded. Binding problem exists, but would support scaleable dispatch of values.
-
-* Could be offered as an AM extension (in provider, ...): scales well
-with #of apps in cluster, but adds initial location/failover problems.
-
-* If offered as a core-YARN service, location is handled via a fixed
-URL. This could place high load on the service, even just 302 redirects.
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/release_notes/release-0.30.0.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/release_notes/release-0.30.0.md b/src/site/markdown/release_notes/release-0.30.0.md
deleted file mode 100644
index 72c6e39..0000000
--- a/src/site/markdown/release_notes/release-0.30.0.md
+++ /dev/null
@@ -1,36 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Release 0.30.0 (incubating)
-
-May 2014
-
-This release is built against the Apache Hadoop 2.4.0, HBase-0.98.1
-and Accumulo 1.5.1 artifacts.
-
-Download: []()
-
-
-## Key changes
-1. Slider application registry that allow registration and discovery of application configuration and URLs (such as jmx endpoints and management UI) for client consumption.
-2. Move to a .zip packaging for Slider application packages.
-3. Richer metainfo support to provide start ordering and arbitrary template that can be published.
-4. Updated application definition packages for HBase, Accumulo, and Storm.
-
-## Other changes
-
-1. [SLIDER-13](https://issues.apache.org/jira/browse/SLIDER-13) switch build to be java7+ only.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/security.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/security.md b/src/site/markdown/security.md
deleted file mode 100644
index 1358662..0000000
--- a/src/site/markdown/security.md
+++ /dev/null
@@ -1,197 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Security
-
-This document discusses the design, implementation and use of Slider
-to deploy secure applications on a secure Hadoop cluster.
-
-### Important:
-
-This document does not cover Kerberos, how to secure a Hadoop cluster, Kerberos
-command line tools or how Hadoop uses delegation tokens to delegate permissions
-round a cluster. These are assumed, though some links to useful pages are
-listed at the bottom.
-
-
-## Concepts
-
-Slider runs in secure clusters, but with restrictions
-
-1. The keytabs to allow a worker to authenticate with the master must
- be distributed in advance: Slider does not attempt to pass these around.
-1. Until the location of Slider node instances can be strictly limited to
- a set of nodes (a future YARN feature), the keytabs must be passed to
- all the nodes in the cluster in advance, *and made available to the
- user creating the cluster*
-1. due to the way that HBase and accumulo authenticate worker nodes to
- the masters, any HBase node running on a server must authenticate as
- the same principal, and so have equal access rights to the HBase cluster.
-1. As the data directories for a slider cluster are created under the home
- directories of that user, the principals representing all role instances
- in the clusters *MUST* have read/write access to these files. This can be
- done with a shortname that matches that of the user, or by requesting
- that Slider create a directory with group write permissions -and using LDAP
- to indentify the application principals as members of the same group
- as the user.
-
-
-## Security Requirements
-
-
-### Needs
-* Slider and deployed applications to work against secure HDFS
-* Slider to work with secure YARN.
-* Slider to start secure applications
-* Kerberos and ActiveDirectory to perform the authentication.
-* Slider to only allow cluster operations by authenticated users -command line and direct RPC.
-* Any Slider Web UI and REST API for Ambari to only allow access to authenticated users.
-* The Slider database in `~/.slider/clusters/$name/data` to be writable by HBase
-
-
-### Short-lived Clusters
-* Cluster to remain secure for the duration of the Kerberos tokens issued to Slider.
-
-
-### Long-lived Clusters
-
-* Slider application instance and HBase instance to remain functional and secure over an indefinite period of time.
-
-### Initial Non-requirements
-* secure audit trail of cluster operations.
-* multiple authorized users being granted rights to a Slider Cluster (YARN admins can always kill the Slider application instance.
-* More than one HBase cluster in the YARN cluster belonging to a single user (irrespective of how they are started).
-* Any way to revoke certificates/rights of running containers.
-
-### Assumptions
-* Kerberos is running and that HDFS and YARN are running Kerberized.
-* LDAP cannot be assumed.
-* Credentials needed for HBase can be pushed out into the local filesystems of
- the of the worker nodes via some external mechanism (e.g. scp), and protected by
- the access permissions of the native filesystem. Any user with access to these
- credentials is considered to have been granted such rights.
-* These credentials can outlive the duration of the HBase containers
-* The user running HBase has the same identity as that of the HBase cluster.
-
-## Design
-
-
-1. The user is expected to have their own Kerberos principal, and have used `kinit`
- or equivalent to authenticate with Kerberos and gain a (time-bounded) TGT
-1. The user is expected to have their own principals for every host in the cluster of the form
- username/hostname@REALM
-1. A keytab must be generated which contains all these principals -and distributed
- to all the nodes in the cluster with read access permissions to the user.
-1. When the user creates a secure cluster, they provide the standard HBase kerberos options
- to identify the principals to use and the keytab location.
-
-The Slider Client will talk to HDFS and YARN authenticating itself with the TGT,
-talking to the YARN and HDFS principals which it has been configured to expect.
-
-This can be done as described in [Client Configuration] (client-configuration.html) on the command line as
-
- -D yarn.resourcemanager.principal=yarn/master@LOCAL
- -D dfs.namenode.kerberos.principal=hdfs/master@LOCAL
-
-The Slider Client will create the cluster data directory in HDFS with `rwx` permissions for
-user `r-x` for the group and `---` for others. (these can be configurable as part of the cluster options),
-
-It will then deploy the AM, which will (somehow? for how long?) retain the access
-rights of the user that created the cluster.
-
-The Application Master will read in the JSON cluster specification file, and instantiate the
-relevant number of componentss.
-
-
-## Securing communications between the Slider Client and the Slider AM.
-
-When the AM is deployed in a secure cluster,
-it automatically uses Kerberos-authorized RPC channels. The client must acquire a
-token to talk the AM.
-
-This is provided by the YARN Resource Manager when the client application
-wishes to talk with the SliderAM -a token which is only provided after
-the caller authenticates itself as the user that has access rights
-to the cluster
-
-To allow the client to freeze a Slider application instance while they are unable to acquire
-a token to authenticate with the AM, use the `--force` option.
-
-### How to enable a secure Slider client
-
-Slider can be placed into secure mode by setting the Hadoop security options:
-
-This can be done in `slider-client.xml`:
-
-
- <property>
- <name>hadoop.security.authorization</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hadoop.security.authentication</name>
- <value>kerberos</value>
- </property>
-
-
-Or it can be done on the command line
-
- -D hadoop.security.authorization=true -D hadoop.security.authentication=kerberos
-
-### Adding Kerberos binding properties to the Slider Client JVM
-
-The Java Kerberos library needs to know the Kerberos controller and
-realm to use. This should happen automatically if this is set up as the
-default Kerberos binding (on a Unix system this is done in `/etc/krb5.conf`.
-
-If is not set up, a stack trace with kerberos classes at the top and
-the message `java.lang.IllegalArgumentException: Can't get Kerberos realm`
-will be printed -and the client will then fail.
-
-The realm and controller can be defined in the Java system properties
-`java.security.krb5.realm` and `java.security.krb5.kdc`. These can be fixed
-in the JVM options, as described in the [Client Configuration] (client-configuration.html)
-documentation.
-
-They can also be set on the Slider command line itself, using the `-S` parameter.
-
- -S java.security.krb5.realm=MINICLUSTER -S java.security.krb5.kdc=hadoop-kdc
-
-### Java Cryptography Exceptions
-
-
-When trying to talk to a secure, cluster you may see the message:
-
- No valid credentials provided (Mechanism level: Illegal key size)]
-
-This means that the JRE does not have the extended cryptography package
-needed to work with the keys that Kerberos needs. This must be downloaded
-from Oracle (or other supplier of the JVM) and installed according to
-its accompanying instructions.
-
-## Useful Links
-
-1. [Adding Security to Apache Hadoop](http://hortonworks.com/wp-content/uploads/2011/10/security-design_withCover-1.pdf)
-1. [The Role of Delegation Tokens in Apache Hadoop Security](http://hortonworks.com/blog/the-role-of-delegation-tokens-in-apache-hadoop-security/)
-1. [Chapter 8. Secure Apache HBase](http://hbase.apache.org/book/security.html)
-1. Hadoop Operations p135+
-1. [Java Kerberos Requirements](http://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/KerberosReq.htmla)
-1. [Troubleshooting Kerberos on Java](http://docs.oracle.com/javase/7/docs/technotes/guides/security/jgss/tutorials/Troubleshooting.html)
-1. For OS/X users, the GUI ticket viewer is `/System/Library/CoreServices/Ticket\ Viewer.app`
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/application_configuration.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/application_configuration.md b/src/site/markdown/slider_specs/application_configuration.md
deleted file mode 100644
index 14acf2d..0000000
--- a/src/site/markdown/slider_specs/application_configuration.md
+++ /dev/null
@@ -1,82 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Application Configuration
-
-App Configuration captures the default configuration associated with the application. *Details of configuration management is discussed in a separate spec*. The default configuration is modified based on user provided InstanceConfiguration, cluster specific details (e.g. HDFS root, local dir root), container allocated resources (port and hostname), and dependencies (e.g. ZK quorom hosts) and handed to the component instances.
-
-App Configuration is a folder containing all configuration needed by the application. Config files include any site.xml, log4j properties file, etc.
-
-In addition, application may have configuration parameters that do not necessarily go into a config files. Such configurations may be used during template expansion (parameters in env.sh files), as environment variables (e.g. JAVA_HOME), customize user names (for runas). These configurations can be provided as user inputs or are automatically inferred from the environment. Such configurations are stored in a file named "app_config.xml".
-
-
-
-A config file is of the form:
-
-
- <?xml version="1.0"?>
- <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- <configuration>
- <property>
- ...
- </property>
- </configuration>
-
-
-
-Each configuration property is specified as follows:
-
-
- <property>
- <name>storm.zookeeper.session.timeout</name>
- <value>20000</value>
- <description>The session timeout for clients to ZooKeeper.</description>
- <required>false</required>
- <valueRestriction>0-30000</valueRestriction>
- </property>
- <property>
- <name>storm.zookeeper.root</name>
- <value>/storm</value>
- <description>The root location at which Storm stores data in ZK.</description>
- <required>true</required>
- </property>
- <property>
- <name>jvm.heapsize</name>
- <value>256</value>
- <description>The default JVM heap size for any component instance.</description>
- <required>true</required>
- </property>
- <property>
- <name>nimbus.host</name>
- <value>localhost</value>
- <description>The host that the master server is running on.</description>
- <required>true</required>
- <clientVisible>true</clientVisible>
- </property>
-
-
-
-* name: name of the parameter
-
-* value: default value of the parameter
-
-* description: a short description of the parameter
-
-* required: if the parameter is mandatory in which case it must have a value - default is "false"
-
-* clientVisible: if the property must be exported for a client
-
[26/50] [abbrv] git commit: SLIDER-153 fix shebang line
Posted by st...@apache.org.
SLIDER-153 fix shebang line
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/77f3e021
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/77f3e021
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/77f3e021
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 77f3e02186cf1e55857eb991c216e94ef4f85c49
Parents: 6fc569d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 13:18:50 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 13:18:50 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/77f3e021/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 1e55cc6..77c39aa 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -1,4 +1,4 @@
-# !/usr/bin/env python
+#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one or more
[20/50] [abbrv] SLIDER-121 removed site documentation from git source
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/application_definition.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/application_definition.md b/src/site/markdown/slider_specs/application_definition.md
deleted file mode 100644
index 85846ac..0000000
--- a/src/site/markdown/slider_specs/application_definition.md
+++ /dev/null
@@ -1,182 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Application Definition
-
-App definition is a declarative definition of a YARN application describing its content. The AppDefinition is used in conjunction with the [AppPackage](application_package.md). Application definition is an XML file and is packaged as metainfo.xml.
-
-## Structure
-
-*Non-mandatory fields are described in **italics**.*
-
-The fields to describe an application is as follows:
-
-* **name**: the name of the application
-
-* **version**: the version of the application. name and version together uniquely identify an application.
-
-* **type**: the type of the application. "YARN-APP" identifies an application definition suitable for YARN.
-
-* **minHadoopVersion**: the minimum version of hadoop on which the app can run
-
-* **components**: the list of component that the application is comprised of
-
-* **osSpecifics**: OS specific package information for the application
-
-* *commandScript*: application wide commands may also be defined. The command is executed on a component instance that is a client
-
-* *dependencies*: application can define a list of dependencies. Dependencies can be on the base services such as HDFS, ZOOKEEPER, YARN which are infrastructure services or GANGLIA, NAGIOS, etc. which are monitoring/alert services. The dependencies are parsed by the management infrastructure to provide the necessary configurations to allow the app to access the services. For example, a HDFS folder could be requested by the app to store its data, a ZOOKEEPER node to co-ordinate among components.
-
-An application contains several component. The fields associated with a component are:
-
-* **name**: name of the component
-
-* **category**: type of the component - MASTER, SLAVE, and CLIENT
-
-* **minInstanceCount**: the minimum number of instances required for this component
-
-* *maxInstanceCount*: maximum number of instances allowed for a component
-
-* **commandScript**: the script that implements the commands.
-
- * **script**: the script location - relative to the AppPackage root
-
- * **scriptType**: type of the script
-
- * **timeout**: default timeout of the script
-
-* *customCommands*: any additional commands available for the component and their implementation
-
-An application definition also includes the package used to install the application. Its typically a tarball or some other form of package that does not require root access to install. The details of what happens during install is captured in the command script.
-
-* **osSpecific**: details on a per OS basis
-
-* **osType**: "any" refers to any OS ~ typical for tarballs
-
-* **packages**: list of packages that needs to be deployed
-
-* **type**: type of package
-
-* **name**: name of the package
-
-* **location**: location of the package (can be a relative folder within the parent AppPackage)
-
-Application can define a an order of activation which dictates if some component activation must follow the successful activation of other components.
-
-* **command**: specifies the component and the command in the form component-command *currently, START is the only valid command*
-
-* **requires**: specifies component and their state that the command depends on, provided in the form component-state *currently, STARTED is the only valid state*
-
-Applications can also advertise a set of properties (typically urls) that can only be bound when the application components are active. One such item can be the jmx endpoint. The properties to be advertised are organized as export groups (exportGroup) and each group can export one or more properties organized as a property bag. These values are visible through the registry service.
-
-* **name**: specifies the name of the export group
-
-Each exportGroup contains one or more exports.
-
-* **name**: the name of the export
-
-* **value**: the template that will be populated by Slider and then exported
-
-
- <metainfo>
- <schemaVersion>2.0</schemaVersion>
- <application>
- <name>HBASE</name>
- <version>0.96.0.2.1.1</version>
- <type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
- <components>
- <component>
- <name>HBASE_MASTER</name>
- <category>MASTER</category>
- <minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
- <commandScript>
- <script>scripts/hbase_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <customCommands>
- <customCommand>
- <name>GRACEFUL_STOP</name>
- <commandScript>
- <script>scripts/hbase_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>1800</timeout>
- </commandScript>
- </customCommand>
- </customCommands>
- </component>
-
- <component>
- <name>HBASE_REGIONSERVER</name>
- <category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
- ...
- </component>
-
- <component>
- <name>HBASE_CLIENT</name>
- <category>CLIENT</category>
- ...
- </components>
-
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>hbase-0.96.1-tar.gz</name>
- <location>package/files</location>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandOrders>
- <commandOrder>
- <command>HBASE_REGIONSERVER-START</command>
- <requires>HBASE_MASTER-STARTED</requires>
- </commandOrder>
- </commandOrders>
-
- <exportGroups>
- <exportGroup>
- <name>QuickLinks</name>
- <exports>
- <export>
- <name>JMX_Endpoint</name>
- <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
- </export>
- <export>
- <name>Master_Status</name>
- <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
- </export>
- </exports>
- </exportGroup>
- </exportGroups>
-
- <commandScript>
- <script>scripts/app_health_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- </application>
- </metainfo>
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/application_instance_configuration.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/application_instance_configuration.md b/src/site/markdown/slider_specs/application_instance_configuration.md
deleted file mode 100644
index d37a3db..0000000
--- a/src/site/markdown/slider_specs/application_instance_configuration.md
+++ /dev/null
@@ -1,104 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider App Instance Configuration
-
-App Instance Configuration is the configuration override provided by the application owner when creating an application instance using Slider. This configuration values override the default configuration available in the App Package.
-
-Instance configuration is a JSON formatted doc in the following form:
-
-
- {
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- },
- }
-
-An appConfig.json contains the application configuration. The sample below shows configuration for HBase.
-
-
- {
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- "config_types": "core-site,hdfs-site,hbase-site",
-
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/hbase-0.96.1-hadoop2-bin.tar",
-
- "site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.security_enabled": "false",
-
- "site.hbase-site.hbase.hstore.flush.retries.number": "120",
- "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
- "site.hbase-site.hbase.hstore.compactionThreshold": "3",
- "site.hbase-site.hbase.rootdir": "${NN_URI}/apps/hbase/data",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}",
- "site.hbase-site.hbase.regionserver.port": "0",
-
- "site.core-site.fs.defaultFS": "${NN_URI}",
- "site.hdfs-site.dfs.namenode.https-address": "${NN_HOST}:50470",
- "site.hdfs-site.dfs.namenode.http-address": "${NN_HOST}:50070"
- }
- }
-
-appConf.json allows you to pass in arbitrary set of configuration that Slider will forward to the application component instances.
-
-**Variable naming convention**
-In order to understand how the naming convention work, lets look at how the config is passed on to component commands. Slider agent recevies a structured bag of commands as input for all commands, INSTALL, CONFIGURE, START, etc. The command includes a section "configuration" which has config properties arranged into named property bags.
-
-* Variables of the form `site.xx.yy` translates to variables by the name `yy` within the group `xx` and are typically converted to site config files by the name `xx` containing variable `yy`. For example, `"site.hbase-site.hbase.regionserver.port":""` will be sent to the Slider-Agent as `"hbase-site" : { "hbase.regionserver.port": ""}` and app definition scripts can access all variables under `hbase-site` as a single property bag.
-* Similarly, `site.core-site.fs.defaultFS` allows you to pass in the default fs. *This specific variable is automatically made available by Slider but its shown here as an example.*
-* Variables of the form `site.global.zz` are sent in the same manner as other site variables except these variables are not expected to get translated to a site xml file. Usually, variables needed for template or other filter conditions (such as security_enabled = true/false) can be sent in as "global variable".
-
-**slider variables**
-
-* Any config not of the form `site.xx.yy` are consumed by Slider itself. Some of the manadatory configuration are:
- * `agent.conf`: location of the agent config file (typically, "/slider/agent/conf/agent.ini")
- * `application.def`: location of the application definition package (typically, "/slider/hbase_v096.zip")
- * `config_types`: list of config types sent to the containers (e.g. "core-site,hdfs-site,hbase-site")
- * `java_home`: java home path (e.g. "/usr/jdk64/jdk1.7.0_45")
- * `package_list`: location of the package relative to the root where AppPackage is installed (e.g. "files/hbase-0.96.1-hadoop2-bin.tar.gz"
-
-**dynamically allocated ports**
-
-Apps can ask port to be dynamically assigned by Slider or they can leave it as "0". If there is a need for advertising any listening endpoint then the ports can be marked such.
-
-For example, HBase master info port needs to be advertised so that jmx endpoint can be accessed. This is indicated by using a special value, of the form, ${COMPONENT_NAME.ALLOCATED_PORT}. E.g. "site.hbase-site.hbase.master.info.port": "${HBASE_MASTER.ALLOCATED_PORT}"
-
-[Application Definition](application_definition.md) describes how to advertise arbitrary set of properties that are dynamically finalized when application is activated.
-
-**configuraing an app for ganglia metrics**
-
-There is no set guideline for doing so. How an application emits metrics and how the metrics are emitted to the right place is completely defined by the application. In the following example, we hso how HBase app is configured to emit metrics to a ganglia server.
-
-Ganglia server lifecycle is not controlled by the app instance. So the app instance only needs to know where to emit the metrics. This is achieved by three global variables
-
-* "site.global.ganglia_server_host": "gangliaserver.my.org"
-* "site.global.ganglia_server_port": "8663"
-* "site.global.ganglia_server_id": "HBaseApplicationCluster3"
-
-All three variable values are user provided. It is also expected that a gmond server is available on host gangliaserver.my.org listening for metrics at port 8663 and is named "HBaseApplicationCluster3". Its the reponsibility of the ganglia server admin to ensure that the server is unique and is only receving metrics from the application instance.
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/application_needs.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/application_needs.md b/src/site/markdown/slider_specs/application_needs.md
deleted file mode 100644
index 10ccf49..0000000
--- a/src/site/markdown/slider_specs/application_needs.md
+++ /dev/null
@@ -1,140 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider's needs of an application
-
-Slider installs and runs applications in a YARN cluster -applications that
-do not need to be written for YARN.
-
-What they do need to be is deployable by Slider, which means installable by YARN,
-configurable by Slider, and, finally, executable by YARN. YARN will kill the
-executed process when destroying a container, so the deployed application
-must expect this to happen and be able to start up from a kill-initiated
-shutdown without any manual recovery process.
-
-They need to locate each other dynamically, both at startup and during execution,
-because the location of processes will be unknown at startup, and may change
-due to server and process failures.
-
-## Must
-
-* Install and run from a tarball -and be run from a user that is not root.
-
-* Be self contained or have all dependencies pre-installed.
-
-* Support dynamic discovery of nodes -such as via ZK.
-
-* Nodes to rebind themselves dynamically -so if nodes are moved, the application
-can continue
-
-* Handle kill as a normal shutdown mechanism.
-
-* Support multiple instances of the application running in the same cluster,
- with processes from different application instances sharing
-the same servers.
-
-* Operate correctly when more than one role instance in the application is
-deployed on the same physical host. (If YARN adds anti-affinity options in
-container requests this will no longer be a requirement)
-
-* Dynamically allocate any RPC or web ports -such as supporting 0 as the number
-of the port to listen on in configuration options.
-
-* Be trusted. YARN does not run code in a sandbox.
-
-* If it talks to HDFS or other parts of Hadoop, be built against/ship with
-libaries compatible with the version of Hadoop running on the cluster.
-
-* Store persistent data in HDFS (directly or indirectly) with the exact storage location
-configurable. Specifically: not to the local filesystem, and not in a hard coded location
-such as `hdfs://app/data`. Slider creates per-instance directories for
-persistent data.
-
-* Be configurable as to where any configuration directory is (or simply relative
-to the tarball). The application must not require it to be in a hard-coded
-location such as `/etc`.
-
-* Not have a fixed location for log output -such as `/var/log/something`
-
-* Run until explicitly terminated. Slider treats an application termination
-(which triggers a container release) as a failure -and reacts to it by restarting
-the container.
-
-
-
-## MUST NOT
-
-* Require human intervention at startup or termination.
-
-## SHOULD
-
-These are the features that we'd like from a service:
-
-* Publish the actual RPC and HTTP ports in a way that can be picked up, such as via ZK
-or an admin API.
-
-* Be configurable via the standard Hadoop mechanisms: text files and XML configuration files.
-If not, custom parsers/configuration generators will be required.
-
-* Support an explicit parameter to define the configuration directory.
-
-* Take late bindings params via -D args or similar
-
-* Be possible to exec without running a complex script, so that process inheritance works everywhere, including (for testing) OS/X
-
-* Provide a way for Slider to get list of nodes in cluster and status. This will let Slider detect failed worker nodes and react to it.
-
-* FUTURE: If a graceful decommissioning is preferred, have an RPC method that a Slider provider can call to invoke this.
-
-* Be location aware from startup. Example: worker nodes to be allocated tables to serve based on which tables are
-stored locally/in-rack, rather than just randomly. This will accelerate startup time.
-
-* Support simple liveness probes (such as an HTTP GET operations).
-
-* Return a well documented set of exit codes, so that failures can be propagated
- and understood.
-
-* Support cluster size flexing: the dynamic addition and removal of nodes.
-
-
-* Support a management platform such as Apache Ambari -so that the operational
-state of a Slider application can be monitored.
-
-## MAY
-
-* Include a single process that will run at a fixed location and whose termination
-can trigger application termination. Such a process will be executed
-in the same container as the Slider AM, and so known before all other containers
-are requested. If a live cluster is unable to handle restart/migration of
-such a process, then the Slider application will be unable to handle
-Slider AM restarts.
-
-* Ideally: report on load/cost of decommissioning.
- E.g amount of data; app load.
-
-
-## MAY NOT
-
-* Be written for YARN.
-
-* Be (pure) Java. If the tarball contains native binaries for the cluster's hardware & OS,
- they should be executable.
-
-* Be dynamically reconfigurable, except for the special requirement of handling
-movement of manager/peer containers in an application-specific manner.
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/application_package.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/application_package.md b/src/site/markdown/slider_specs/application_package.md
deleted file mode 100644
index 0469e6c..0000000
--- a/src/site/markdown/slider_specs/application_package.md
+++ /dev/null
@@ -1,147 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider App Package
-
-All application artifacts, app definition, app configuration, scripts are packaged into a structured single package that can be handed off to any YARN application deployment tool including Slider
-
-## Overall Structure
-
-App package is a zip package containing all application artifacts. App package contains the following items:
-
-* **app definition file**
-application structure, content, definition, supported platforms, version, etc.
-
-* **default configurations folder**
-various configurations and configuration files associated with the application
-
-* **cmd_impl folder**
-management operations for the application/component
-
- * **scripts folder**
-various scripts that implement management operations
-
- * **templates folder**
-various templates used by the application
-
- * **files folder**
-other scripts, txt files, tarballs, etc.
-
-
-
-
-The example above shows a semi-expanded view of an application "HBASE-YARN-APP" and the package structure for OOZIE command scripts.
-
-## app definition
-
-App definition is a file named "metainfo.xml". The file contains application definition as described in [Application Definition](application_definition.md).
-
-## default configurations
-
-This folder consists of various config files containing default configuration as described in [App Configuration](application_configuration.md).
-
-## package folder
-
-package includes the "implementation" of all management operations. The folders within are divided into scripts, templates, and files.
-
-### scripts folder
-
-Scripts are the implementation of management operations. There are five default operations and a composite operation. "restart" can be redefined to have a custom implementation.
-
-1. install
-
-2. configure
-
-3. start
-
-4. stop
-
-5. status
-
-6. restart (by default calls stop + start)
-
-The script specified in the metainfo is expected to understand the command. It can choose to call other scripts based on how the application author organizes the code base. For example:
-
-
- class OozieServer(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- oozie(is_server=True)
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- oozie_service(action='start')
-
- def stop(self, env):
- import params
- env.set_params(params)
- oozie_service(action='stop')
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- check_process_status(status_params.pid_file)
-
-
-
-The scripts are invoked in the following manner:
-
- python SCRIPT COMMAND JSON_FILE PACKAGE_ROOT STRUCTURED_OUT_FILE
-
-* SCRIPT is the top level script that implements the commands for the component.
-
-* COMMAND is one of the six commands listed above or can be a custom command as defined in Application Definition
-
-* JSON_FILE includes all configuration parameters and the values
-
-* PACKAGE_ROOT is the root folder of the package. From this folder, its possible to access files, scripts, templates, packages (e.g. tarballs), etc. The App author has complete control over the structure of the package
-
-* STRUCTURED_OUT_FILE is the file where the script can output structured data. The management infrastructure is expected to automatically reports back STD_OUT and STD_ERR.
-
-A separate document (link TBD) discusses how the scripts are developed and the structure of the JSON_FILE containing the parameters.
-
-### templates folder
-
-templates are configurable text files that are NOT regular config files. *A library has been developed that can materialize a complete site configuration file from a property bag and therefore are not generated from templates.* Other files such as env sh files, log4j properties file, etc. may be derived from a template. Again, the implementor can choose to create these files from scratch and not use templates. The following operations are allowed during template expansion:
-
-* variable expansion
-
-* if condition
-
-* for loop
-
-* ...
-
-Sample template file for dfs.exclude file to list excluded/decommissioned hosts. hdfs_exclude_files in the property defined in params.py which is populated from config parameters defined in JSON_FILE.
-
- {% if hdfs_exclude_file %}
- {% for host in hdfs_exclude_file %}
- {{host}}
- {% endfor %}
- {% endif %}
-
-
-### files folder
-
-files is a directory to store any other files that are needed for management operations. Sample files stored here are tarballs used to install the application, shell scripts used by various operations.
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/canonical_scenarios.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/canonical_scenarios.md b/src/site/markdown/slider_specs/canonical_scenarios.md
deleted file mode 100644
index a877d97..0000000
--- a/src/site/markdown/slider_specs/canonical_scenarios.md
+++ /dev/null
@@ -1,165 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Guidelines for Clients and Client Applications
-
-This document will define the canonical scenarios for the deployment and management of Slider hosted applications. It will define the types of applications supported, the sequence of events for deploying the application types, and the management facilities exposed for the deployed applications.
-
-## Deployable Application Types
-
-The server-side components of an application (alternatively referred to as the application components) will be deployed and managed using a fairly uniform mechanism. However, applications can be distinguished by their associated client implementation. Specifically, each different client application type can yield different development, deployment, and management approaches. There are two primary application client types:
-
-1. **New application client** - the application deployed to Slider will interact with a newly coded application client leveraging the Slider supplied client API and management facilities. The client API will make use of distributed management facilities such as Zookeeper to provide the application client with the configuration information required for invoking remote application components. For example, the application client (or an application hosted component, e.g. a web application) will leverage the API to lookup the appropriate host(s) and port(s) for RPC based communication. Alternatively, if annotation libraries or an "app container" environment is provided, the appropriate values will be injected into the client process.
-
-2. **Existing, configurable client** - the application client predates deployment in the Slider environment, but is implemented in a manner that can be integrated with the Slider application client support facilities (APIs etc). This case is probably very similar in nature to the new application client in terms of the mechanisms it will use for component discovery, but is distinguished by the fact that it’s development pre-dates Slider. There are two possible variants of application client in this category:
-
- 1. A client that is static - the client is dependent on existing configuration properties to communicate with master components and the existing code can not been altered (at least in the short term). This type of client would require a support infrastructure that may be relatively complex (e.g. templating of configuration files, proxying of server components).
-
- 2. A client that can be enhanced - a client that can have its code altered can leverage a number of mechanisms (REST APIs, Zookeeper, a provided client discover API) to obtain the information required to invoke the master components of the application.
-
-## Deployment Scenarios
-
-There are two primary deployment mechanisms to examine: application component and client-side (application client) component deployment.
-
-## Application Component Deployment
-
-Applications generally are composed of one or more components. In the deployment steps below, be advised that there may be a need to repeat some of the configuration/definition steps for each component.
-
-The process of deploying applications (specifically, the non-client components of an application) is:
-
-1. Compose an application package that contains:
-
- 1. An application definition that provides the following items:
-
- 1. name and version of application
-
- 2. component type(s) and role(s)
-
- 3. system requirements (RAM, CPUs, disk space etc)
-
- 4. ports required for RPC, UI
-
- 5. software dependencies (HDP deployed services required in the cluster, JDK versions, python versions, etc)
-
- 6. configurable properties including:
-
- 1. application-specific properties. If the properties designate port numbers, the general recommendation would be to set them to zero to allow for the system to assign a dynamically selected port number that subsequently can be published to zookeeper. Other properties designating remote components running on other hosts may need some additional support (floating IPs, discovery APIs, etc). The processing of these properties by the infrastructure requires agreed upon mechanisms for identification (e.g. all properties beginning with the application name are passed through as application properties with the application name prefix stripped)
-
- 2. Slider-specific properties. The following Slider properties are currently available:
-
- 1. yarn.memory
-
- 2. Environment variables specified with a "env." prefix (e.g. env.MALLOC_ARENA_MAX)
-
- 3. role.instances
-
- 4. role.name
-
- 5. yarn.vcores
-
- 2. Other application artifacts (binaries etc)
-
-2. Install the application
-
- 1. The application package will be made available in a designated HDFS location
-
- 2. If there is a managed application client component it will be deployed to selected nodes as defined in the cluster specification.
-
- 3. Slider interacts with yarn (resource manager and node manager(s)) to populate the local resources on the designated nodes.
-
- 4. Some properties important for facilitating remote interaction with the deployed components are advertised via zookeeper (though this may actually take place during component start up as the assignment of ports is a late-binding operation. Alternatively, developers may be encouraged to store these values in a registry rather than as direct-application configuration properties).
-
-## Client Application Deployment
-
-Although application clients are components that are deployed using mechanisms similar to other application components (especially in the managed case), there are a number of features that distinguish them:
-
-1. **configuration** - client applications generally require some information (host names, ports, etc) to ascertain the correct networking values required to communicate with the application's server components. In order to work in a yarn deployment, this configuration may need to be manipulated to allow proper operation (e.g. the configuration files updated with correct values, the configuration properties ascertained dynamically from services such as Zookeeper)
-
-2. **execution context **- it may be necessary to provide an execution environment for application clients that allows for discovery mechanisms (dependency injection, annotation libraries, etc)
-
-For each of these application client types there are two possible deployment modes:
-
-* **Managed** - the application client is deployed via Slider mechanisms. Clients, in this context, differ from the other application components in that they are not running, daemon processes. However, in a managed environment there is the expectation that the appropriate binaries and application elements will be distributed to the designated client hosts, and the configuration on those hosts will be updated to allow for execution of requests to the application’s master/server components. Therefore, client components should be defined in the application specification as elements that the management infrastructure supports (Figure 1).
-
-
-Figure 1 - Managed Application Client and associated Slider Application
-
-* **Unmanaged** - the application client is run as a process outside of Slider/yarn, although it may leverage Slider provided libraries that allow for server component discovery etc (Figure 2). These libraries would primarily be client bindings providing access to the registry leveraged by Slider (e.g. Java and python bindings to Zookeeper)
-
-
-Figure 2 - Unmanaged Application Client and associated Slider Application
-
-### Managed Application Client
-
-A managed application client is a component defined as part of the Slider/yarn application (i.e. it is part of the application definition presented to Slider). As such, it is deployed and managed via standard Slider/yarn mechanisms. This type of application client is more than likely already configured and written to work in a yarn environment.
-
-There are two primary needs to be met for a properly functioning client:
-
-1. **Discovery** - as a client, it is important that the client application retrieve the information it requires in order to communicate with the remote application components. As application components are spawned they (or the associated container agent) will advertise the relevant information using zookeeper. It will be up to the client (or the associated Slider client library) to contact zookeeper and retrieve the requested information.
-
-2. **Configuration** - there may be use cases in which a large number of configuration items are required by the client for its processing. In such cases it is more appropriate for a client to perform a bulk download of the application component(s) configuration as a JSON or XML document (via zookeeper or Slider-app comm?)
-
-Whether ascertained via discovery or bulk configuration retrieval, the attributes that the client obtains will more than likely need to be populated into the client’s configuration files. Therefore, a templating facility or the like should be provided to allow for such configuration file manipulation.
-
-### Unmanaged Application Client
-
-An unmanaged application client is a standalone application that leverages application components deployed into the Slider/yarn environment. It is not possible to predict the deployment mechanisms or network topology employed for an unmanaged application. However, it is feasible to provide some guidance and/or software (APIs etc) to allow for application component discovery and communication.
-
-## Application Management
-
-Post deployment, the Slider infrastructure will provide the requisite set of administrative facilities required for application management, including the ability to start/stop the application, monitor the application, and reconfigure the application.
-
-### General Management
-
-There is one general management command:
-
-* List Yarn Apps - returns a listing of deployed yarn apps and associated information:
-
- * name and version
-
- * dependencies (required HDP services and versions, etc)
-
- * configuration properties
-
- * components/roles and associated configuration
-
-### Application/Component Management
-
-The following administrative functions are supported for applications:
-
-* Install the application - the installation command will take care of the population of the application resources into the pre-determined application resource directory in HDFS. The required configuration and binary directories will also be created.
-
-* start/thaw the application - Slider/Yarn runtime negotiates and instantiates the number of component containers designated by the cluster description and the components are started.
-
-* stop/freeze the application - similar to stopping, applicaiton (or a subset of their components) can be stopped.
-
-* get application status - the retrieval of application status may take a number of forms, including:
-
- * liveness of service components
-
- * operational metrics (application-level or component-level)
-
- * viewing of logs
-
-* get application configuration - the configuration of application components is retrieved (JSON or XML form)
-
-* get cluster configuration - the cluster configuration is retrieved (number of various application components, associated hosts etc)
-
-* get cluster history
-
-* re-configure cluster
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/creating_app_definitions.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/creating_app_definitions.md b/src/site/markdown/slider_specs/creating_app_definitions.md
deleted file mode 100644
index ab9f51c..0000000
--- a/src/site/markdown/slider_specs/creating_app_definitions.md
+++ /dev/null
@@ -1,135 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider AppPackage
-
-Slider AppPackages are a declarative definition of an application for application management. AppPackage is not a packaging scheme for application binaries and artifacts. Tarball, zip files, rpms etc. are available for that purpose. Instead AppPackage includes the application binaries along with other artifacts necessary for application management.
-
-An application instance consists of several active component such as one or more masters and several slaves. There may be a number of accompanying processes in addition to the basic master/slave processes - lets refer to all processes as app component instances. When run in the context of Yarn, the application specific processes are activated within individual Yarn Container. If you pry into an Yarn container (created through Slider) it will be apparent as to what is the role of Slider-Agent and the actual application components. The following image provides an high-level view. Within a container there are at least two processes - and instance of a slider agent and an instance of an application component. The application can itself spawn more procsses if needed.
-
-
-
-Figure 1 - High-level view of a container
-
-For example:
-
- yarn 8849 -- python ./infra/agent/slider-agent/agent/main.py --label container_1397675825552_0011_01_000003___HBASE_REGIONSERVER --host AM_HOST --port 47830
- yarn 9085 -- bash /hadoop/yarn/local/usercache/yarn/appcache/application_1397675825552_0011/ ... internal_start regionserver
- yarn 9114 -- /usr/jdk64/jdk1.7.0_45/bin/java -Dproc_regionserver -XX:OnOutOfMemoryError=...
-
-The above list shows three processes, the Slider-Agent process, the bash script to start HBase Region Server and the HBase Region server itself. *Three of these together constitute the container*.
-
-## Using an AppPackage
-The following command creates an HBase application using the AppPackage for HBase.
-
- ./slider create cl1 --image hdfs://NN:8020/slider/agent/slider-agent.tar.gz --template /work/appConf.json --resources /work/resources.json
-
-Lets analyze various parameters from the perspective of app creation:
-
-* `--image`: its the slider agent tarball
-* `--template`: app configuration
-* `--resources`: yarn resource requests
-* … other parameters are described in accompanying docs.
-
-### AppPackage
-The structure of an AppPackage is described at [AppPackage](application_package.md).
-
-In the enlistment, there are three example AppPackages:
-
-* `app-packages/hbase-v0_96`
-* `app-packages/accumulo`
-* `app-packages/storm-v0_91`
-
-The above folders, with minor edits, can be packaged as *zip* files to get the corresponding AppPackages. The application tarball file, containing the binaries/artifacts of the application itself is a component within the AppPackage. They are:
-
-* For hbase - `app-packages/hbase-v0_96/package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE`
-* For accumulo - `app-packages/accumulo/package/files/` (no file present)
-* For storm - `app-packages/storm-v0_91/package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.placeholder`
-
-**They are placehoder files**, mostly because the files themselves are too large as well as users are free to use their own version of the package. To create a Slider AppPackage - replace the file with an actual application tarball and then ensure that the metainfo.xml has the correct file name. After that create a zip file using standard zip commands and ensure that the package has the metainfo.xml file at the root folder.
-
-For Accumulo, this is automated in maven by running
-
- mvn clean package -DskipTests -Paccumulo-app-package
-
-Additional configuration parameters are described in `app-packages/accumulo/README.txt`.
-
-For other app packages, follow this example:
-
-* cd slider/app-packages/hbase
-* zip -r hbase-v098.zip .
-* Looking at the content through unzip -l "$@" hbase-v098.zip
-
-```
-Archive: hbase-v098.zip
- Length Date Time Name
---------- ---------- ----- ----
- 3158 06-19-2014 22:08 appConfig.json
- 0 06-19-2014 22:15 configuration/
- 5077 06-19-2014 22:15 configuration/global.xml
- 5248 06-19-2014 22:15 configuration/hbase-log4j.xml
- 2250 06-19-2014 22:15 configuration/hbase-policy.xml
- 14705 06-19-2014 22:15 configuration/hbase-site.xml
- 2142 06-19-2014 22:17 jmx_metrics.json
- 3991 06-19-2014 22:45 metainfo.xml
- 0 06-19-2014 22:33 package/
- 0 06-19-2014 22:31 package/files/
- 83278591 06-19-2014 22:31 package/files/hbase-0.98.3-hadoop2-bin.tar.gz
- 0 06-19-2014 22:29 package/scripts/
- 787 06-19-2014 22:27 package/scripts/__init__.py
- 1378 06-19-2014 22:27 package/scripts/functions.py
- 3738 06-19-2014 22:27 package/scripts/hbase.py
- 1205 06-19-2014 22:27 package/scripts/hbase_client.py
- 1640 06-19-2014 22:27 package/scripts/hbase_master.py
- 1764 06-19-2014 22:27 package/scripts/hbase_regionserver.py
- 1482 06-19-2014 22:27 package/scripts/hbase_service.py
- 5089 06-19-2014 22:27 package/scripts/params.py
- 973 06-19-2014 22:27 package/scripts/status_params.py
- 0 06-19-2014 22:34 package/templates/
- 2799 06-19-2014 22:34 package/templates/hadoop-metrics2-hbase.properties-GANGLIA-MASTER.j2
- 2799 06-19-2014 22:34 package/templates/hadoop-metrics2-hbase.properties-GANGLIA-RS.j2
- 3878 06-19-2014 22:34 package/templates/hbase-env.sh.j2
- 909 06-19-2014 22:34 package/templates/hbase_client_jaas.conf.j2
- 989 06-19-2014 22:34 package/templates/hbase_master_jaas.conf.j2
- 1001 06-19-2014 22:34 package/templates/hbase_regionserver_jaas.conf.j2
- 837 06-19-2014 22:34 package/templates/regionservers.j2
- 3486 06-19-2014 22:39 pom.xml
- 1921 06-19-2014 21:53 README.txt
- 358 06-19-2014 22:36 resources.json
- 0 06-19-2014 22:40 src/
- 0 06-19-2014 23:02 src/assembly/
- 2400 06-19-2014 22:45 src/assembly/hbase.xml
---------- -------
- 83354595 35 files
-```
-
-Sample **resources.json** and **appConfig.json** files are also included in the enlistment. These are samples and are typically tested on one node test installations.
-
-
-### --template appConfig.json
-An appConfig.json contains the application configuration. See [Specifications InstanceConfiguration](application_instance_configuration.md) for details on how to create a template config file. The enlistment includes sample config files for HBase, Accumulo, and Storm.
-
-
-### --resources resources.json
-Resource specification is an input to Slider to specify the Yarn resource needs for each component type that belong to the application. [Specification of Resources](resource_specification.html) describes how to write a resource config json file. The enlistment includes sample config files for HBase, Accumulo, and Storm.
-
-
-## Scripting for AppPackage
-Refer to [App Command Scripts](writing_app_command_scripts) for details on how to write scripts for a AppPackage. These scripts are in the package/script folder within the AppPackage. *Use the checked in samples for HBase/Storm/Accumulo as reference for script development.*
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/index.md b/src/site/markdown/slider_specs/index.md
deleted file mode 100644
index 953229e..0000000
--- a/src/site/markdown/slider_specs/index.md
+++ /dev/null
@@ -1,53 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider
-
-##Introduction
-
-**SLIDER: A collection of tools and technologies to simplify the packaging, deployment and management of long-running applications on YARN.**
-
-- Availability (always-on) - YARN works with the application to ensure recovery or restart of running application components.
-- Flexibility (dynamic scaling) - YARN provides the application with the facilities to allow for scale-up or scale-down
-- Resource Mgmt (optimization) - YARN handles allocation of cluster resources.
-
-## Terminology
-
-### Apps on YARN
- - Application written to run directly on YARN
- - Packaging, deployment and lifecycle management are custom built for each application
-
-### Slider Apps
- - Applications deployed and managed on YARN using Slider
- - Use of slider minimizes custom code for deployment + lifecycle management
- - Requires apps to follow Slider guidelines and packaging ("Sliderize")
-
-Specifications
----
-
-The entry points to leverage Slider are:
-
-- [Application Needs](application_needs.html) What it takes to be deployable by Slider.
-- [Slider AppPackage](creating_app_definitions.html) Overview of how to create an Slider AppPackage.
-- [Specifications for AppPackage](application_package.html) Describes the structure of an AppPackage
-- [Specifications for Application Definition](application_definition.html) How to write metainfo.xml?
-- [Specifications for Configuration](application_configuration.html) How to create a template config file for an app?
-- [Specification of Resources](resource_specification.html) How to write a resource spec for an app?
-- [Specifications InstanceConfiguration](application_instance_configuration.html) How to write a template config for an app?
-- [Guidelines for Clients and Client Applications](canonical_scenarios.html)
-- [Documentation for "General Developer Guidelines"](../developing/index.html)
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/resource_specification.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/resource_specification.md b/src/site/markdown/slider_specs/resource_specification.md
deleted file mode 100644
index cdcfb02..0000000
--- a/src/site/markdown/slider_specs/resource_specification.md
+++ /dev/null
@@ -1,53 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Resource Specification
-Resource specification is an input to Slider to specify the Yarn resource needs for each component type that belong to the application.
-
-An example resource requirement for an application that has two components "master" and "worker" is as follows. Slider will automatically add the requirements for the AppMaster for the application. This compoent is named "slider-appmaster".
-
-Some parameters that can be specified for a component instance include:
-
-* `yarn.memory`: amount of memory requried for the component instance
-* `yarn.vcores`: number of vcores requested
-* `yarn.role.priority`: each component must be assigned unique priority. Component with higher priority come up earlier than components with lower priority
-* `yarn.component.instances`: number of instances for this component type
-
-Sample:
-
- {
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- },
- "components" : {
- "HBASE_MASTER" : {
- "yarn.role.priority" : "1",
- "yarn.component.instances" : "1"
- "yarn.memory" : "768",
- "yarn.vcores" : "1"
- },
- "slider-appmaster" : {
- },
- "HBASE_REGIONSERVER" : {
- "yarn.role.priority" : "2",
- "yarn.component.instances" : "1"
- }
- }
- }
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/slider_specs/writing_app_command_scripts.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/slider_specs/writing_app_command_scripts.md b/src/site/markdown/slider_specs/writing_app_command_scripts.md
deleted file mode 100644
index e59f0b9..0000000
--- a/src/site/markdown/slider_specs/writing_app_command_scripts.md
+++ /dev/null
@@ -1,211 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: Developing App Command Scripts
-
-App command implementations follow a standard structure so that they can be invoked in an uniform manner. For any command, the python scripts are invoked as:
-
- python SCRIPT COMMAND JSON_FILE PACKAGE_ROOT STRUCTURED_OUT_FILE
-
-* SCRIPT is the top level script that implements the commands for the component.
-
-* COMMAND is one of the following default commands - START, STOP, INSTALL, CONFIG, RESTART, STATUS or any custom commands.
-
-* JSON_FILE includes all configuration parameters and the values.
-
-* PACKAGE_ROOT is the root folder of the package. From this folder, its possible to access files, scripts, templates, packages (e.g. tarballs), etc. The Yarn-App author has complete control over the structure of the package as long as the PACKAGE_ROOT and SCRIPT path is known to the management tool.
-
-* STRUCTURED_OUT_FILE is the file where the script can output structured data.
-
-The management infrastructure is expected to automatically reports back STD_OUT and STD_ERR.
-
-Sample:
-
- python /apps/HBASE_ON_YARN/package/scripts/hbase_regionserver.py START /apps/commands/cmd_332/command.json /apps/HBASE_ON_YARN/package /apps/commands/cmd_332/strout.txt
-
-**Note**: The above is how Slider-Agent invokes the scripts. Its provided as a reference for developing the scripts themselves as well as a way to test/debug the scripts.
-
-## Structure of JSON formatted parameter
-
-The parameters are organized as multi-layer name-value pairs.
-
- {
- "commandId": "Command Id as assigned by Slider",
- "command": "Command being executed",
- "commandType": "Type of command",
- "clusterName": "Name of the cluster",
- "appName": "Name of the app",
- "component": "Name of the component",
- "hostname": "Name of the host",
- "public_hostname": "FQDN of the host",
- "hostParams": {
- "host specific parameters common to all commands"
- },
- "componentParams": {
- "component specific parameters, if any"
- },
- "commandParams": {
- "command specific parameters, usually used in case of custom commands"
- },
- "configurations": {
- "app-global-config": {
- },
- "config-type-2": {
- },
- "config-type-2": {
- }
- }
- }
-
-
-## Sample configuration parameters
-
- {
- "commandId": "2-2",
- "command": "START",
- "commandType": "EXECUTION_COMMAND",
- "clusterName": "c1",
- "appName": "HBASE",
- "componentName": "HBASE_MASTER",
- "hostParams": {
- "java_home": "/usr/jdk64/jdk1.7.0_45"
- },
- "componentParams": {},
- "commandParams": {},
- "hostname": "c6403.ambari.apache.org",
- "public_hostname": "c6403.ambari.apache.org",
- "configurations": {
- "hbase-log4j": {
- "log4j.threshold": "ALL",
- "log4j.rootLogger": "${hbase.root.logger}",
- "log4j.logger.org.apache.zookeeper": "INFO",
- "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
- "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": "INFO",
- "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
- "log4j.category.SecurityLogger": "${hbase.security.logger}",
- "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
- "log4j.appender.console.target": "System.err",
- "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
- "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: %m%n",
- "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
- "log4j.appender.RFAS.MaxBackupIndex": "${hbase.security.log.maxbackupindex}",
- "log4j.appender.RFAS.File": "${hbase.log.dir}/${hbase.security.log.file}",
- "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
- "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
- "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
- "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
- "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
- "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
- "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
- "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
- "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
- "log4j.appender.DRFA.DatePattern": ".yyyy-MM-dd",
- "log4j.additivity.SecurityLogger": "false",
- "hbase.security.logger": "INFO,console",
- "hbase.security.log.maxfilesize": "256MB",
- "hbase.security.log.maxbackupindex": "20",
- "hbase.security.log.file": "SecurityAuth.audit",
- "hbase.root.logger": "INFO,console",
- "hbase.log.maxfilesize": "256MB",
- "hbase.log.maxbackupindex": "20",
- "hbase.log.file": "hbase.log",
- "hbase.log.dir": "."
- },
- "app-global-config": {
- "security_enabled": "false",
- "pid_dir": "/hadoop/yarn/log/application_1394053491953_0003/run",
- "log_dir": "/hadoop/yarn/log/application_1394053491953_0003/log",
- "tmp_dir": "/hadoop/yarn/log/application_1394053491953_0003/tmp",
- "user_group": "hadoop",
- "user": "hbase",
- "hbase_regionserver_heapsize": "1024m",
- "hbase_master_heapsize": "1024m",
- "fs_default_name": "hdfs://c6403.ambari.apache.org:8020",
- "hdfs_root": "/apps/hbase/instances/01",
- "zookeeper_node": "/apps/hbase/instances/01",
- "zookeeper_quorom_hosts": "c6403.ambari.apache.org",
- "zookeeper_port": "2181",
- },
- "hbase-site": {
- "hbase.hstore.flush.retries.number": "120",
- "hbase.client.keyvalue.maxsize": "10485760",
- "hbase.hstore.compactionThreshold": "3",
- "hbase.rootdir": "hdfs://c6403.ambari.apache.org:8020/apps/hbase/instances/01/data",
- "hbase.stagingdir": "hdfs://c6403.ambari.apache.org:8020/apps/hbase/instances/01/staging",
- "hbase.regionserver.handler.count": "60",
- "hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "hbase.hregion.memstore.block.multiplier": "2",
- "hbase.hregion.memstore.flush.size": "134217728",
- "hbase.superuser": "yarn",
- "hbase.zookeeper.property.clientPort": "2181",
- "hbase.regionserver.global.memstore.upperLimit": "0.4",
- "zookeeper.session.timeout": "30000",
- "hbase.tmp.dir": "/hadoop/yarn/log/application_1394053491953_0003/tmp",
- "hbase.hregion.max.filesize": "10737418240",
- "hfile.block.cache.size": "0.40",
- "hbase.security.authentication": "simple",
- "hbase.defaults.for.version.skip": "true",
- "hbase.zookeeper.quorum": "c6403.ambari.apache.org",
- "zookeeper.znode.parent": "/apps/hbase/instances/01",
- "hbase.hstore.blockingStoreFiles": "10",
- "hbase.hregion.majorcompaction": "86400000",
- "hbase.security.authorization": "false",
- "hbase.cluster.distributed": "true",
- "hbase.hregion.memstore.mslab.enabled": "true",
- "hbase.client.scanner.caching": "100",
- "hbase.zookeeper.useMulti": "true",
- "hbase.regionserver.info.port": "",
- "hbase.master.info.port": "60010"
- }
- }
- }
-
-
-## Sample command script
-
- class OozieServer(Script):
- def install(self, env):
- self.install_packages(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- oozie(is_server=True)
-
- def start(self, env):
- import params
- env.set_params(params)
- self.configure(env)
- oozie_service(action='start')
-
- def stop(self, env):
- import params
- env.set_params(params)
- oozie_service(action='stop')
-
- def status(self, env):
- import status_params
- env.set_params(status_params)
- check_process_status(status_params.pid_file)
-
-
-
[09/50] [abbrv] git commit: SLIDER-159 Jenkins testEcho failing
Posted by st...@apache.org.
SLIDER-159 Jenkins testEcho failing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/537d87c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/537d87c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/537d87c2
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 537d87c25e20e3c11994dbfdcfcb89ba20f59859
Parents: bc0c672
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 13:25:41 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 13:25:41 2014 -0700
----------------------------------------------------------------------
.../services/workflow/ForkedProcessService.java | 25 ++++++++++++++++----
.../services/workflow/LongLivedProcess.java | 13 ++++++++++
.../TestWorkflowForkedProcessService.java | 3 ++-
3 files changed, 35 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/537d87c2/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
index b0c503d..7e73005 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -269,15 +269,28 @@ public class ForkedProcessService extends AbstractWorkflowExecutorService implem
/**
* Get the recent output from the process, or [] if not defined
- * @param duration the duration, in ms, which we wait for recent output to become non-empty
+ *
+ * @param finalOutput flag to indicate "wait for the final output of the process"
+ * @param duration the duration, in ms,
+ * ro wait for recent output to become non-empty
* @return a possibly empty list
*/
- public List<String> getRecentOutput(int duration) {
- if (process == null) return new LinkedList<String>();
+ public List<String> getRecentOutput(boolean finalOutput, int duration) {
+ if (process == null) {
+ return new LinkedList<String>();
+ }
long start = System.currentTimeMillis();
- while (process.isRecentOutputEmpty() && System.currentTimeMillis() - start <= duration) {
+ while (System.currentTimeMillis() - start <= duration) {
+ if (finalOutput && process.isFinalOutputProcessed()) {
+ //end of stream, break
+ break;
+ }
+ if (!process.isRecentOutputEmpty()) {
+ // there is some output
+ break;
+ }
try {
- Thread.sleep(20);
+ Thread.sleep(100);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
break;
@@ -285,4 +298,6 @@ public class ForkedProcessService extends AbstractWorkflowExecutorService implem
}
return process.getRecentOutput();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/537d87c2/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index 7b9863f..d9ddecb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -35,6 +35,7 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* Execute a long-lived process.
@@ -82,6 +83,7 @@ public class LongLivedProcess implements Runnable {
private final List<String> recentLines = new LinkedList<String>();
private int recentLineLimit = RECENT_LINE_LOG_LIMIT;
private LongLivedProcessLifecycleEvent lifecycleCallback;
+ private final AtomicBoolean finalOutputProcessed = new AtomicBoolean(false);
/**
@@ -347,6 +349,14 @@ public class LongLivedProcess implements Runnable {
}
/**
+ * Query to see if the final output has been processed
+ * @return
+ */
+ public boolean isFinalOutputProcessed() {
+ return finalOutputProcessed.get();
+ }
+
+ /**
* add the recent line to the list of recent lines; deleting
* an earlier on if the limit is reached.
*
@@ -499,6 +509,9 @@ public class LongLivedProcess implements Runnable {
LOG.warn("encountered {}", ignored, ignored);
//process connection has been torn down
} finally {
+ //mark output as done
+ finalOutputProcessed.set(true);
+ // close streams
IOUtils.closeStream(errReader);
IOUtils.closeStream(outReader);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/537d87c2/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
index d46f07c..6d08156 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowForkedProcessService.java
@@ -43,6 +43,7 @@ public class TestWorkflowForkedProcessService extends WorkflowServiceTestBase {
private static final Logger
processLog =
LoggerFactory.getLogger("org.apache.hadoop.services.workflow.Process");
+ public static final int RECENT_OUTPUT_SLEEP_DURATION = 4000;
private ForkedProcessService process;
@@ -118,7 +119,7 @@ public class TestWorkflowForkedProcessService extends WorkflowServiceTestBase {
* @return the last output
*/
private List<String> getFinalOutput() {
- return process.getRecentOutput(2000);
+ return process.getRecentOutput(true, RECENT_OUTPUT_SLEEP_DURATION);
}
private ForkedProcessService initProcess(List<String> commands) throws
[41/50] [abbrv] git commit: SLIDER-96. Ensure documents and
metainfo.xml use/refer to consistent XML structure - rooted at application
Posted by st...@apache.org.
SLIDER-96. Ensure documents and metainfo.xml use/refer to consistent XML structure - rooted at application
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4eb8ea99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4eb8ea99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4eb8ea99
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 4eb8ea99970b8b7dd5d284809c43ab71dc2be9fc
Parents: e4c855a
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jun 24 18:12:11 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jun 26 13:41:31 2014 -0700
----------------------------------------------------------------------
.../providers/agent/AgentClientProvider.java | 14 +-
.../slider/providers/agent/AgentKeys.java | 1 -
.../providers/agent/AgentProviderService.java | 52 +++----
.../agent/application/metadata/Application.java | 121 +++++++++++++++++
.../agent/application/metadata/Metainfo.java | 17 +--
.../application/metadata/MetainfoParser.java | 10 +-
.../agent/application/metadata/Service.java | 121 -----------------
.../slider/common/tools/TestSliderUtils.java | 23 ++--
.../agent/TestAgentClientProvider.java | 40 +++++-
.../agent/TestAgentProviderService.java | 61 ++++-----
.../metadata/MetainfoParserTest.java | 15 +-
.../org/apache/slider/tools/TestUtility.java | 71 ++++++++++
.../org/apache/slider/common/tools/test.zip | Bin 1273 -> 0 bytes
.../slider/common/tools/test/metainfo.txt | 16 +++
.../slider/common/tools/test/metainfo.xml | 95 +++++++++++++
.../slider/common/tools/test/someOtherFile.txt | 16 +++
.../slider/common/tools/test/someOtherFile.xml | 16 +++
.../agent/application/metadata/metainfo.xml | 136 +++++++++----------
18 files changed, 532 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 1946ebd..ea1c1b7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -34,8 +34,8 @@ import org.apache.slider.core.launch.AbstractLauncher;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.Service;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -152,7 +152,7 @@ public class AgentClientProvider extends AbstractClientProvider
instanceDefinition.getAppConfOperations().
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
} catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided." + bce.getMessage());
+ throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
}
String appDef = instanceDefinition.getAppConfOperations().
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
@@ -171,7 +171,7 @@ public class AgentClientProvider extends AbstractClientProvider
throw new BadConfigException("Either agent package path " +
AgentKeys.PACKAGE_PATH + " or image root " +
OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH
- + " must be provided");
+ + " must be provided.");
}
try {
@@ -204,11 +204,11 @@ public class AgentClientProvider extends AbstractClientProvider
Set<String> tags;
try {
Metainfo metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef);
- Service service = metainfo.getServices().get(0);
+ Application application = metainfo.getApplication();
tags = new HashSet<>();
- tags.add("Name: " + service.getName());
- tags.add("Version: " + service.getVersion());
- tags.add("Description: " + SliderUtils.truncate(service.getComment(), 80));
+ tags.add("Name: " + application.getName());
+ tags.add("Version: " + application.getVersion());
+ tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
} catch (IOException e) {
log.error("error retrieving metainfo from {}", appDef, e);
throw new SliderException("error retrieving metainfo", e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index dea39ea..333058d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -86,7 +86,6 @@ public interface AgentKeys {
String JAVA_HOME = "java_home";
String PACKAGE_LIST = "package_list";
- String COMPONENT_SCRIPT = "role.script";
String WAIT_HEARTBEAT = "wait.heartbeat";
String PYTHON_EXE = "python";
String HEARTBEAT_MONITOR_INTERVAL = "heartbeat.monitor.interval";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 85945d8..a685a3e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -50,11 +50,11 @@ import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.Service;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -184,12 +184,12 @@ public class AgentProviderService extends AbstractProviderService implements
initializeAgentDebugCommands(instanceDefinition);
metainfo = getApplicationMetainfo(fileSystem, appDef);
- if (metainfo == null || metainfo.getServices() == null || metainfo.getServices().size() == 0) {
+ if (metainfo == null || metainfo.getApplication() == null) {
log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
throw new SliderException("metainfo.xml is required in app package.");
}
- commandOrder = new ComponentCommandOrder(metainfo.getServices().get(0).getCommandOrder());
+ commandOrder = new ComponentCommandOrder(metainfo.getApplication().getCommandOrder());
monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
monitor.start();
}
@@ -574,8 +574,8 @@ public class AgentProviderService extends AbstractProviderService implements
publishComponentConfiguration(key, key, configs.entrySet());
}
- Service service = getMetainfo().getServices().get(0);
- List<ExportGroup> exportGroups = service.getExportGroups();
+ Application application = getMetainfo().getApplication();
+ List<ExportGroup> exportGroups = application.getExportGroups();
if (exportGroups != null && !exportGroups.isEmpty()) {
String configKeyFormat = "${site.%s.%s}";
@@ -624,17 +624,18 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Extract script path from the application metainfo
+ *
* @param roleName
+ *
* @return
*/
protected String getScriptPathFromMetainfo(String roleName) {
String scriptPath = null;
- List<Service> services = getMetainfo().getServices();
- if (services.size() != 1) {
- log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ Application application = getMetainfo().getApplication();
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
}
- Service service = services.get(0);
- for (Component component : service.getComponents()) {
+ for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
scriptPath = component.getCommandScript().getScript();
break;
@@ -645,16 +646,17 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Is the role of type MASTER
+ *
* @param roleName
+ *
* @return
*/
protected boolean isMaster(String roleName) {
- List<Service> services = getMetainfo().getServices();
- if (services.size() != 1) {
- log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ Application application = getMetainfo().getApplication();
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
} else {
- Service service = services.get(0);
- for (Component component : service.getComponents()) {
+ for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
if (component.getCategory().equals("MASTER")) {
return true;
@@ -669,16 +671,17 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Can the role publish configuration
+ *
* @param roleName
+ *
* @return
*/
protected boolean canPublishConfig(String roleName) {
- List<Service> services = getMetainfo().getServices();
- if (services.size() != 1) {
- log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ Application application = getMetainfo().getApplication();
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
} else {
- Service service = services.get(0);
- for (Component component : service.getComponents()) {
+ for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
return Boolean.TRUE.toString().equals(component.getPublishConfig());
}
@@ -693,12 +696,11 @@ public class AgentProviderService extends AbstractProviderService implements
*/
protected boolean canAnyMasterPublishConfig() {
if (canAnyMasterPublish == null) {
- List<Service> services = getMetainfo().getServices();
- if (services.size() != 1) {
- log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ Application application = getMetainfo().getApplication();
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
} else {
- Service service = services.get(0);
- for (Component component : service.getComponents()) {
+ for (Component component : application.getComponents()) {
if (Boolean.TRUE.toString().equals(component.getPublishConfig()) &&
component.getCategory().equals("MASTER")) {
canAnyMasterPublish = true;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
new file mode 100644
index 0000000..b007313
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Application type defined in the metainfo
+ */
+public class Application {
+ String name;
+ String comment;
+ String version;
+ List<Component> components;
+ List<ExportGroup> exportGroups;
+ List<OSSpecific> osSpecifics;
+ List<CommandOrder> commandOrders;
+ ConfigurationDependencies configDependencies;
+
+ public Application() {
+ exportGroups = new ArrayList<>();
+ components = new ArrayList<>();
+ osSpecifics = new ArrayList<>();
+ commandOrders = new ArrayList<>();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public ConfigurationDependencies getConfigDependencies() {
+ return configDependencies;
+ }
+
+ public void setConfigDependencies(ConfigurationDependencies configDependencies) {
+ this.configDependencies = configDependencies;
+ }
+
+ public void addComponent(Component component) {
+ components.add(component);
+ }
+
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public void addExportGroup(ExportGroup exportGroup) {
+ exportGroups.add(exportGroup);
+ }
+
+ public List<ExportGroup> getExportGroups() {
+ return exportGroups;
+ }
+
+ public void addOSSpecific(OSSpecific osSpecific) {
+ osSpecifics.add(osSpecific);
+ }
+
+ public List<OSSpecific> getOSSpecifics() {
+ return osSpecifics;
+ }
+
+ public void addCommandOrder(CommandOrder commandOrder) {
+ commandOrders.add(commandOrder);
+ }
+
+ public List<CommandOrder> getCommandOrder() {
+ return commandOrders;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append(",\n\"name\": ").append(name);
+ sb.append(",\n\"comment\": ").append(comment);
+ sb.append(",\n\"version\" :").append(version);
+ sb.append(",\n\"components\" : {");
+ for (Component component : components) {
+ sb.append("\n").append(component);
+ }
+ sb.append("\n},");
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
index 21e8b24..2455e8e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -16,19 +16,14 @@
*/
package org.apache.slider.providers.agent.application.metadata;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
/**
- *
+ * Application metainfo uber class
*/
public class Metainfo {
String schemaVersion;
- List<Service> services;
+ Application application;
public Metainfo() {
- services = new ArrayList<Service>();
}
public String getSchemaVersion() {
@@ -39,11 +34,11 @@ public class Metainfo {
this.schemaVersion = schemaVersion;
}
- public void addService(Service service) {
- services.add(service);
+ public Application getApplication() {
+ return application;
}
- public List<Service> getServices() {
- return Collections.unmodifiableList(services);
+ public void setApplication(Application application) {
+ this.application = application;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index a97c879..c7922a7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -34,10 +34,10 @@ public class MetainfoParser {
digester.addObjectCreate("metainfo", Metainfo.class);
digester.addBeanPropertySetter("metainfo/schemaVersion");
- digester.addObjectCreate("*/service", Service.class);
- digester.addBeanPropertySetter("*/service/name");
- digester.addBeanPropertySetter("*/service/comment");
- digester.addBeanPropertySetter("*/service/version");
+ digester.addObjectCreate("*/application", Application.class);
+ digester.addBeanPropertySetter("*/application/name");
+ digester.addBeanPropertySetter("*/application/comment");
+ digester.addBeanPropertySetter("*/application/version");
digester.addObjectCreate("*/commandOrder", CommandOrder.class);
digester.addBeanPropertySetter("*/commandOrder/command");
@@ -79,7 +79,7 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/config-type", "configType");
digester.addSetNext("*/configuration-dependencies", "setConfigDependencies");
- digester.addSetNext("*/service", "addService");
+ digester.addSetRoot("*/application", "setApplication");
try {
return (Metainfo) digester.parse(metainfoStream);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java
deleted file mode 100644
index 0fc009f..0000000
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Service.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class Service {
- String name;
- String comment;
- String version;
- List<Component> components;
- List<ExportGroup> exportGroups;
- List<OSSpecific> osSpecifics;
- List<CommandOrder> commandOrders;
- ConfigurationDependencies configDependencies;
-
- public Service() {
- exportGroups = new ArrayList<>();
- components = new ArrayList<>();
- osSpecifics = new ArrayList<>();
- commandOrders = new ArrayList<>();
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public ConfigurationDependencies getConfigDependencies() {
- return configDependencies;
- }
-
- public void setConfigDependencies(ConfigurationDependencies configDependencies) {
- this.configDependencies = configDependencies;
- }
-
- public void addComponent(Component component) {
- components.add(component);
- }
-
- public List<Component> getComponents() {
- return components;
- }
-
- public void addExportGroup(ExportGroup exportGroup) {
- exportGroups.add(exportGroup);
- }
-
- public List<ExportGroup> getExportGroups() {
- return exportGroups;
- }
-
- public void addOSSpecific(OSSpecific osSpecific) {
- osSpecifics.add(osSpecific);
- }
-
- public List<OSSpecific> getOSSpecifics() {
- return osSpecifics;
- }
-
- public void addCommandOrder(CommandOrder commandOrder) {
- commandOrders.add(commandOrder);
- }
-
- public List<CommandOrder> getCommandOrder() {
- return commandOrders;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"name\": ").append(name);
- sb.append(",\n\"comment\": ").append(comment);
- sb.append(",\n\"version\" :").append(version);
- sb.append(",\n\"components\" : {");
- for (Component component : components) {
- sb.append("\n").append(component);
- }
- sb.append("\n},");
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
index 7cac439..be850da 100644
--- a/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ b/slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
@@ -19,23 +19,30 @@ package org.apache.slider.common.tools;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.junit.Test;
+import org.apache.slider.tools.TestUtility;
import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.InputStream;
-import java.net.URI;
-/**
- *
- */
+/** Test slider util methods. */
public class TestSliderUtils {
protected static final Logger log =
LoggerFactory.getLogger(TestSliderUtils.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
@Test
- public void testGetMetaInfoStreamFromZip () throws Exception {
+ public void testGetMetaInfoStreamFromZip() throws Exception {
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.getLocal(configuration);
log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
@@ -43,14 +50,14 @@ public class TestSliderUtils {
InputStream stream = SliderUtils.getApplicationResourceInputStream(
sliderFileSystem.getFileSystem(),
- new Path("target/test-classes/org/apache/slider/common/tools/test.zip"),
+ new Path(zipFileName),
"metainfo.xml");
Assert.assertTrue(stream != null);
Assert.assertTrue(stream.available() > 0);
}
@Test
- public void testTruncate () {
+ public void testTruncate() {
Assert.assertEquals(SliderUtils.truncate(null, 5), null);
Assert.assertEquals(SliderUtils.truncate("323", -1), "323");
Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
index 69f5a1c..4cb35aa 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
@@ -18,27 +18,41 @@ package org.apache.slider.providers.agent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.tools.TestUtility;
+import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.InputStream;
import java.util.Set;
/**
*
*/
public class TestAgentClientProvider {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentClientProvider.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
@Test
- public void testGetApplicationTags () throws Exception {
+ public void testGetApplicationTags() throws Exception {
Configuration configuration = new Configuration();
FileSystem fs = FileSystem.getLocal(configuration);
SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
AgentClientProvider provider = new AgentClientProvider(null);
- Set<String> tags = provider.getApplicationTags(sliderFileSystem,
- "target/test-classes/org/apache/slider/common/tools/test.zip");
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
+ Set<String> tags = provider.getApplicationTags(sliderFileSystem, zipFileName);
assert tags != null;
assert !tags.isEmpty();
assert tags.contains("Name: STORM");
@@ -46,4 +60,18 @@ public class TestAgentClientProvider {
assert tags.contains("Version: 0.9.1.2.1");
}
+
+ @Test
+ public void testValidateInstanceDefinition() throws Exception {
+ AgentClientProvider provider = new AgentClientProvider(null);
+ AggregateConf instanceDefinition = new AggregateConf();
+
+ try {
+ provider.validateInstanceDefinition(instanceDefinition);
+ Assert.assertFalse("Should fail with BadConfigException", true);
+ } catch (BadConfigException e) {
+ log.info(e.toString());
+ Assert.assertTrue(e.getMessage().contains("Application definition must be provided"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 918fda3..9e9e0db 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -43,13 +43,13 @@ import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.Service;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.apache.slider.server.appmaster.model.mock.MockFileSystem;
import org.apache.slider.server.appmaster.model.mock.MockNodeId;
@@ -97,8 +97,7 @@ public class TestAgentProviderService {
LoggerFactory.getLogger(TestAgentProviderService.class);
private static final String metainfo_1_str = "<metainfo>\n"
+ " <schemaVersion>2.0</schemaVersion>\n"
- + " <services>\n"
- + " <service>\n"
+ + " <application>\n"
+ " <name>HBASE</name>\n"
+ " <comment>\n"
+ " Apache HBase\n"
@@ -164,15 +163,11 @@ public class TestAgentProviderService {
+ " </packages>\n"
+ " </osSpecific>\n"
+ " </osSpecifics>\n"
- + " </service>\n"
- + " </services>\n"
+ + " </application>\n"
+ "</metainfo>";
-
-
private static final String metainfo_2_str = "<metainfo>\n"
+ " <schemaVersion>2.0</schemaVersion>\n"
- + " <services>\n"
- + " <service>\n"
+ + " <application>\n"
+ " <name>HBASE</name>\n"
+ " <comment>\n"
+ " Apache HBase\n"
@@ -203,9 +198,9 @@ public class TestAgentProviderService {
+ " </commandScript>\n"
+ " </component>\n"
+ " </components>\n"
- + " </service>\n"
- + " </services>\n"
+ + " </application>\n"
+ "</metainfo>";
+
@Test
public void testRegistration() throws IOException {
@@ -244,7 +239,7 @@ public class TestAgentProviderService {
doReturn(access).when(mockAps).getAmState();
doReturn("scripts/hbase_master.py").when(mockAps).getScriptPathFromMetainfo(anyString());
Metainfo metainfo = new Metainfo();
- metainfo.addService(new Service());
+ metainfo.setApplication(new Application());
doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
try {
@@ -262,7 +257,6 @@ public class TestAgentProviderService {
desc.setInfo(StatusKeys.INFO_AM_HOSTNAME, "host1");
desc.setInfo(StatusKeys.INFO_AM_WEB_PORT, "8088");
desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
- desc.getOrAddRole("HBASE_MASTER").put(AgentKeys.COMPONENT_SCRIPT, "scripts/hbase_master.py");
expect(access.getClusterStatus()).andReturn(desc).anyTimes();
AggregateConf aggConf = new AggregateConf();
@@ -352,7 +346,7 @@ public class TestAgentProviderService {
public void testProcessConfig() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
- assert metainfo.getServices().size() == 1;
+ Assert.assertNotNull(metainfo.getApplication());
AgentProviderService aps = new AgentProviderService();
HeartBeat hb = new HeartBeat();
ComponentStatus status = new ComponentStatus();
@@ -405,12 +399,12 @@ public class TestAgentProviderService {
public void testMetainfoParsing() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
- Assert.assertEquals(metainfo.getServices().size(), 1);
- Service service = metainfo.getServices().get(0);
- log.info("Service: " + service.toString());
- Assert.assertEquals(service.getName(), "HBASE");
- Assert.assertEquals(service.getComponents().size(), 2);
- List<Component> components = service.getComponents();
+ Assert.assertNotNull(metainfo.getApplication());
+ Application application = metainfo.getApplication();
+ log.info("Service: " + application.toString());
+ Assert.assertEquals(application.getName(), "HBASE");
+ Assert.assertEquals(application.getComponents().size(), 2);
+ List<Component> components = application.getComponents();
int found = 0;
for (Component component : components) {
if (component.getName().equals("HBASE_MASTER")) {
@@ -430,8 +424,8 @@ public class TestAgentProviderService {
}
Assert.assertEquals(found, 2);
- assert service.getExportGroups().size() == 1;
- List<ExportGroup> egs = service.getExportGroups();
+ assert application.getExportGroups().size() == 1;
+ List<ExportGroup> egs = application.getExportGroups();
ExportGroup eg = egs.get(0);
assert eg.getName().equals("QuickLinks");
assert eg.getExports().size() == 2;
@@ -441,20 +435,20 @@ public class TestAgentProviderService {
if (export.getName().equals("JMX_Endpoint")) {
found++;
Assert.assertEquals(export.getValue(),
- "http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx");
+ "http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx");
}
if (export.getName().equals("Master_Status")) {
found++;
Assert.assertEquals(export.getValue(),
- "http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status");
+ "http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status");
}
}
Assert.assertEquals(found, 2);
- List<CommandOrder> cmdOrders = service.getCommandOrder();
+ List<CommandOrder> cmdOrders = application.getCommandOrder();
Assert.assertEquals(cmdOrders.size(), 2);
found = 0;
- for (CommandOrder co : service.getCommandOrder()) {
+ for (CommandOrder co : application.getCommandOrder()) {
if (co.getCommand().equals("HBASE_REGIONSERVER-START")) {
Assert.assertTrue(co.getRequires().equals("HBASE_MASTER-STARTED"));
found++;
@@ -660,14 +654,16 @@ public class TestAgentProviderService {
cr.setRole("HBASE_REGIONSERVER");
cr.setRoleCommand("INSTALL");
cr.setStatus("COMPLETED");
- cr.setFolders(new HashMap<String, String>() {{put("a", "b");}});
+ cr.setFolders(new HashMap<String, String>() {{
+ put("a", "b");
+ }});
hb.setReports(Arrays.asList(cr));
hbr = mockAps.handleHeartBeat(hb);
Assert.assertEquals(3, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
- anyString(),
- any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ any(HeartBeatResponse.class),
+ anyString());
// RS still does not start
hb = new HeartBeat();
hb.setResponseId(3);
@@ -723,8 +719,8 @@ public class TestAgentProviderService {
hb.setReports(Arrays.asList(cr));
mockAps.handleHeartBeat(hb);
Mockito.verify(mockAps, Mockito.times(1)).addGetConfigCommand(anyString(),
- anyString(),
- any(HeartBeatResponse.class));
+ anyString(),
+ any(HeartBeatResponse.class));
// RS starts now
hb = new HeartBeat();
@@ -746,7 +742,6 @@ public class TestAgentProviderService {
anyCollection());
}
-
@Test
public void testAddStartCommand() throws Exception {
AgentProviderService aps = new AgentProviderService();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
index ac1bd81..61c53df 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,16 +40,16 @@ public class MetainfoParserTest {
METAINFO_XML);
MetainfoParser parser = new MetainfoParser();
Metainfo metainfo = parser.parse(resStream);
- assert metainfo != null;
- assert metainfo.services.size() == 1;
- Service service = metainfo.getServices().get(0);
- assert "STORM".equals(service.getName());
- assert 5 == service.getComponents().size();
- OSPackage pkg = service.getOSSpecifics().get(0).getPackages().get(0);
+ Assert.assertNotNull(metainfo);
+ Assert.assertNotNull(metainfo.getApplication());
+ Application application = metainfo.getApplication();
+ assert "STORM".equals(application.getName());
+ assert 5 == application.getComponents().size();
+ OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
assert "tarball".equals(pkg.getType());
assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
boolean found = false;
- for (Component comp : service.getComponents()) {
+ for (Component comp : application.getComponents()) {
if (comp != null && comp.getName().equals("NIMBUS")) {
found = true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java b/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
new file mode 100644
index 0000000..a8b14ac
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.tools;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/** Various utility methods */
+public class TestUtility {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestUtility.class);
+
+ public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException {
+ for (File file : dirObj.listFiles()) {
+ if (file.isDirectory()) {
+ addDir(file, zipFile, prefix + file.getName() + File.separator);
+ } else {
+ log.info("Adding to zip - " + prefix + file.getName());
+ zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
+ IOUtils.copy(new FileInputStream(file), zipFile);
+ zipFile.closeArchiveEntry();
+ }
+ }
+ }
+
+ public static void zipDir(String zipFile, String dir) throws IOException {
+ File dirObj = new File(dir);
+ ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile));
+ log.info("Creating : " + zipFile);
+ try {
+ addDir(dirObj, out, "");
+ } finally {
+ out.close();
+ }
+ }
+
+ public static String createAppPackage(
+ TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException {
+ String zipFileName;
+ File pkgPath = folder.newFolder(subDir);
+ File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
+ zipFileName = zipFile.getAbsolutePath();
+ TestUtility.zipDir(zipFileName, srcPath);
+ log.info("Created temporary zip file at {}", zipFileName);
+ return zipFileName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/resources/org/apache/slider/common/tools/test.zip
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/common/tools/test.zip b/slider-core/src/test/resources/org/apache/slider/common/tools/test.zip
deleted file mode 100644
index 18acf1c..0000000
Binary files a/slider-core/src/test/resources/org/apache/slider/common/tools/test.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt b/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt
new file mode 100644
index 0000000..a1d7780
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt
@@ -0,0 +1,16 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
new file mode 100644
index 0000000..3d24f96
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
@@ -0,0 +1,95 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
+ <components>
+
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configuration-dependencies>
+ <config-type>storm-site</config-type>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+ </application>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt b/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt
new file mode 100644
index 0000000..a1d7780
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt
@@ -0,0 +1,16 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml b/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml
new file mode 100644
index 0000000..a1d7780
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml
@@ -0,0 +1,16 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4eb8ea99/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
index 2fcf4cd..3d24f96 100644
--- a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
@@ -18,80 +18,78 @@
<metainfo>
<schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>STORM</name>
- <comment>Apache Hadoop Stream processing framework</comment>
- <version>0.9.1.2.1</version>
- <components>
+ <application>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
+ <components>
- <component>
- <name>NIMBUS</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/nimbus.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>STORM_REST_API</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/rest_api.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>SUPERVISOR</name>
- <category>SLAVE</category>
- <commandScript>
- <script>scripts/supervisor.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>STORM_UI_SERVER</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/ui_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>DRPC_SERVER</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/drpc_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
- <configuration-dependencies>
- <config-type>storm-site</config-type>
- <config-type>global</config-type>
- </configuration-dependencies>
- </service>
- </services>
+ <configuration-dependencies>
+ <config-type>storm-site</config-type>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+ </application>
</metainfo>
[44/50] [abbrv] git commit: SLIDER-176. Container request after
failure is failing to allocate replacement container
Posted by st...@apache.org.
SLIDER-176. Container request after failure is failing to allocate replacement container
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ea251f16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ea251f16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ea251f16
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: ea251f165bf322c4cd86c6d84483169b07b6fba5
Parents: 78a1364
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jun 26 13:41:26 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jun 26 13:41:31 2014 -0700
----------------------------------------------------------------------
.../appmaster/state/OutstandingRequest.java | 2 +-
.../app_packages/test_command_log/metainfo.xml | 62 ++++++++++----------
2 files changed, 31 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ea251f16/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 45dd9d1..7d3e427 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -108,7 +108,7 @@ public final class OutstandingRequest {
if (node != null) {
hosts = new String[1];
hosts[0] = node.hostname;
- relaxLocality = false;
+ relaxLocality = true;
locationSpecified = true;
// tell the node it is in play
node.getOrCreate(roleId);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ea251f16/slider-core/src/test/app_packages/test_command_log/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/metainfo.xml b/slider-core/src/test/app_packages/test_command_log/metainfo.xml
index f5fde30..845ad93 100644
--- a/slider-core/src/test/app_packages/test_command_log/metainfo.xml
+++ b/slider-core/src/test/app_packages/test_command_log/metainfo.xml
@@ -17,38 +17,36 @@
-->
<metainfo>
<schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>TEST_COMMAND_LOG</name>
- <comment>
- When started it creates a new log file and stores all commands in the
- log file. When stopped it renames the file.
- </comment>
- <version>0.1.0</version>
- <components>
- <component>
- <name>COMMAND_LOGGER</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/cl.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
+ <application>
+ <name>TEST_COMMAND_LOG</name>
+ <comment>
+ When started it creates a new log file and stores all commands in the
+ log file. When stopped it renames the file.
+ </comment>
+ <version>0.1.0</version>
+ <components>
+ <component>
+ <name>COMMAND_LOGGER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/cl.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/command_log.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/command_log.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
- </service>
- </services>
+ </application>
</metainfo>
[48/50] [abbrv] git commit: SLIDER-173 Fix typo
Posted by st...@apache.org.
SLIDER-173 Fix typo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a048c9f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a048c9f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a048c9f5
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: a048c9f59c8252905e3b3a9a87ea7ceb49f50cf7
Parents: 581c182
Author: tedyu <yu...@gmail.com>
Authored: Fri Jun 27 11:40:25 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Jun 27 11:40:25 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/providers/agent/AgentProviderService.java | 1 +
.../apache/slider/server/appmaster/state/ContainerPriority.java | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a048c9f5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a685a3e..85aa8db 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -634,6 +634,7 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetainfo().getApplication();
if (application == null) {
log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
+ return scriptPath;
}
for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a048c9f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
index 935c09f..56a5af2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
@@ -43,7 +43,7 @@ public final class ContainerPriority {
public static int buildPriority(int role,
boolean locationSpecified) {
int location = locationSpecified ? LOCATION : 0;
- return role | LOCATION;
+ return role | location;
}
[31/50] [abbrv] git commit: SLIDER-160: move test for file existence
into upload method; now covers all source files
Posted by st...@apache.org.
SLIDER-160: move test for file existence into upload method; now covers all source files
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eba97da0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eba97da0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eba97da0
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: eba97da01765ecf924d0a4f672e8ffd2474b7464
Parents: 5464584
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 17:42:36 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 17:42:36 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/funtest/framework/AgentUploads.groovy | 2 --
.../org/apache/slider/funtest/framework/FileUploader.groovy | 3 +++
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eba97da0/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
index 5f10c0e..3570183 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -48,8 +48,6 @@ class AgentUploads implements FuntestProperties {
*/
def uploadAgentFiles(File tarballDir, boolean force) {
def localAgentTar = new File(tarballDir, AGENT_SLIDER_GZ_IN_SLIDER_TAR)
- assert localAgentTar.exists()
-
def agentTarballPath = new Path(
homeDir,
AGENT_TAR_FILENAME)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eba97da0/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index 262693c..4f61730 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -44,6 +44,9 @@ class FileUploader {
* @return
*/
public boolean copyIfOutOfDate(File src, Path destPath, boolean force) {
+ if (!src.exists()) {
+ throw new FileNotFoundException("Source file $src not found")
+ }
def srcLen = src.length()
def fs = getFileSystem(destPath)
boolean toCopy = force
[34/50] [abbrv] git commit: SLIDER-169 Change proxy-user cookie log
to DEBUG
Posted by st...@apache.org.
SLIDER-169 Change proxy-user cookie log to DEBUG
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/93bd8ce6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/93bd8ce6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/93bd8ce6
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 93bd8ce6f3ef3ab7e94a4ab7298c321549de1565
Parents: 1866733
Author: tedyu <yu...@gmail.com>
Authored: Wed Jun 25 10:24:42 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Jun 25 10:24:42 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/web/SliderAmIpFilter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/93bd8ce6/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
index dec89d1..fc96284 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
@@ -121,7 +121,7 @@ public class SliderAmIpFilter implements Filter {
}
try {
if (user == null) {
- log.warn("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ log.debug("Could not find " + WebAppProxyServlet.PROXY_USER_COOKIE_NAME
+ " cookie, so user will not be set");
chain.doFilter(req, resp);
} else {
[17/50] [abbrv] git commit: SLIDER-153 python script working: stdout
and stderr is being printed after the end of the process
Posted by st...@apache.org.
SLIDER-153 python script working: stdout and stderr is being printed after the end of the process
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/06816c4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/06816c4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/06816c4c
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 06816c4cd6ffbb94014ff95250f26c5028723a09
Parents: acdfe5b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 10:52:10 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 10:52:10 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 28 ++++++++++++++++---------
1 file changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/06816c4c/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 2c1f4b3..cfad59a 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -97,21 +97,27 @@ def main():
print "slider_home = \"%s\"" % slider_home
print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
print "slider_classpath = \"%s\"" % slider_classpath
-
-
- commandline = ["java",]
+
+ #java = "/usr/bin/java"
+ java = "java"
+ commandline = [java,
+ "-classpath",
+ slider_classpath,
+ SLIDER_CLASSNAME]
# commandline.append(slider_jvm_opts)
- commandline.append("-classpath")
- commandline.append(slider_classpath)
- commandline.append(SLIDER_CLASSNAME)
commandline.extend(args)
print "ready to exec : %s" % commandline
# docs warn of using PIPE on stderr
- return subprocess.call(commandline,
- stdin=None,
+ exe = subprocess.Popen(commandline,
+ stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
+ stderr=subprocess.PIPE,
shell=False)
+ exe.wait()
+ out, err = exe.communicate()
+ print "stdout : ", out.decode()
+ print "stderr : ", err.decode()
+ return exe.returncode
@@ -122,7 +128,9 @@ if __name__ == '__main__':
try:
rv = main()
if rv != 0:
- print "exit code = %d" % rv
+ print "Failed with exit code = %d" % rv
+ else:
+ print "Success"
except Exception as e:
print "Exception: %s " % e.message
rv = -1
[29/50] [abbrv] git commit: SLIDER-153 bin/slider is now python
Posted by st...@apache.org.
SLIDER-153 bin/slider is now python
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6f05995e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6f05995e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6f05995e
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 6f05995e42c767062500da68759f75e7f66fd05c
Parents: 8a8ffd9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 16:08:58 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 16:08:58 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 217 +++++++++++++++++-------
slider-assembly/src/main/scripts/slider.py | 169 ------------------
slider-assembly/src/main/scripts/slider.sh | 74 ++++++++
3 files changed, 230 insertions(+), 230 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f05995e/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
old mode 100755
new mode 100644
index caf275b..d48eca6
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -1,5 +1,6 @@
-#!/usr/bin/env bash
-
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -7,68 +8,162 @@
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-
-# this is the shell script to start Slider deploying an application
-# Usage: slider <action> <commands>
-
-# The env variable SLIDER_JVM_OPTS can be used to override
-# the default JVM opts
-
-function usage
-{
- echo "Usage: slider <action> <arguments>"
- echo ""
-}
-
-# Slider works out its own location
-this="${BASH_SOURCE-$0}"
-bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
-script="$(basename -- "$this")"
-
-# lib directory is one up; it is expected to contain
-# slider.jar and any other dependencies that are not in the
-# standard Hadoop classpath
-
-slider_home="${bindir}/.."
-slider_home=`cd -P "${slider_home}" && pwd -P`
-
-libdir="${slider_home}/lib"
-libdir=`cd -P "${libdir}" && pwd -P`
-
-
-confdir="${slider_home}/conf"
-
-# normalize the conf dir so it can be passed down
-confdir=`cd -P "${confdir}" && pwd -P`
-confdir=${SLIDER_CONF_DIR:-$confdir}
-
-
-slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}"
-slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
-
-# allow for an extra classpath
-slider_classpath_extra=${SLIDER_CLASSPATH_EXTRA:-""}
-
-slider_classpath="${libdir}/*:${confdir}:${slider_classpath_extra}"
-
-launcher=org.apache.slider.Slider
-
-
-echo "slider_home = \"${slider_home}\""
-echo "slider_jvm_opts = \"${slider_jvm_opts}\""
-echo "classpath = \"${slider_classpath}\""
-export CLASSPATH="${slider_classpath}"
-echo ""
-
-echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\" ${launcher} $@"
-echo ""
-echo ""
-exec java ${slider_jvm_opts} ${launcher} $@
+import sys
+import os
+import subprocess
+
+CONF = "conf"
+
+LIB = "lib"
+
+SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
+SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
+SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
+
+SLIDER_CLASSNAME = "org.apache.slider.Slider"
+DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
+
+"""
+Launches slider
+
+
+"""
+
+
+
+def scriptDir():
+ """
+ get the script path
+ """
+ return os.path.dirname(os.path.realpath(__file__))
+
+def sliderDir():
+ return os.path.dirname(scriptDir())
+
+def libDir(sliderdir) :
+ return os.path.join(sliderdir, LIB)
+
+def confDir(sliderdir):
+ """
+ determine the active configuration directory
+ :param sliderdir: slider directory
+ :return: the configuration directory -any env var will
+ override the relative path
+ """
+ localconf = os.path.join(sliderdir, CONF)
+ return os.environ.get(SLIDER_CONF_DIR,localconf)
+
+def dirMustExist(dir):
+ if not os.path.exists(dir):
+ raise Exception("Directory does not exist: %s " % dir)
+ return dir
+
+def read(pipe, line):
+ """
+ read a char, append to the listing if there is a char that is not \n
+ :param pipe: pipe to read from
+ :param line: line being built up
+ :return: (the potentially updated line, flag indicating newline reached)
+ """
+
+ c = pipe.read(1)
+ if c != "":
+ o = c.decode('utf-8')
+ if o != '\n':
+ line += o
+ return line, False
+ else:
+ return line, True
+ else:
+ return line, False
+
+
+
+def usage():
+ print "Usage: slider <action> <arguments>"
+ return 1
+
+
+def main():
+ """
+ Slider main method
+ :return: exit code of the process
+ """
+ if len(sys.argv)==1 :
+ return usage()
+ print "stdout encoding: "+ sys.stdout.encoding
+ args = sys.argv[1:]
+ slider_home = sliderDir()
+ libdir = dirMustExist(libDir(slider_home))
+ confdir = dirMustExist(confDir(slider_home))
+ default_jvm_opts = DEFAULT_JVM__OPTS % confdir
+ slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
+ # split the JVM opts by space
+ jvm_opts_split = slider_jvm_opts.split()
+ slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
+ p = os.pathsep # path separator
+ d = os.sep # dir separator
+ slider_classpath = libdir + d + "*" + p \
+ + confdir + p \
+ + slider_classpath_extra
+
+
+ print "slider_home = \"%s\"" % slider_home
+ print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
+ print "slider_classpath = \"%s\"" % slider_classpath
+
+ #java = "/usr/bin/java"
+ commandline = ["java", ]
+ commandline.append("-classpath")
+ commandline.append(slider_classpath)
+ commandline.extend(jvm_opts_split)
+ commandline.append(SLIDER_CLASSNAME)
+ commandline.extend(args)
+ print "ready to exec : %s" % commandline
+ # docs warn of using PIPE on stderr
+ exe = subprocess.Popen(commandline,
+ stdin=None,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=False)
+ stdout = exe.stdout
+ stderr = exe.stderr
+ outline = ""
+ errline = ""
+ while exe.poll() is None:
+ # process is running; grab output and echo every line
+ outline, done = read(stdout, outline)
+ if done:
+ print outline
+ outline = ""
+ errline, done = read(stderr, errline)
+ if done:
+ print errline
+ errline = ""
+
+ # get tail
+ out, err = exe.communicate()
+ print outline + out.decode()
+ print errline + err.decode()
+ return exe.returncode
+
+
+
+if __name__ == '__main__':
+ """
+ Entry point
+ """
+ try:
+ returncode = main()
+ except Exception as e:
+ print "Exception: %s " % e.message
+ returncode = -1
+
+ sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f05995e/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
deleted file mode 100644
index d48eca6..0000000
--- a/slider-assembly/src/main/scripts/slider.py
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import sys
-import os
-import subprocess
-
-CONF = "conf"
-
-LIB = "lib"
-
-SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
-SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
-SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
-
-SLIDER_CLASSNAME = "org.apache.slider.Slider"
-DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
-
-"""
-Launches slider
-
-
-"""
-
-
-
-def scriptDir():
- """
- get the script path
- """
- return os.path.dirname(os.path.realpath(__file__))
-
-def sliderDir():
- return os.path.dirname(scriptDir())
-
-def libDir(sliderdir) :
- return os.path.join(sliderdir, LIB)
-
-def confDir(sliderdir):
- """
- determine the active configuration directory
- :param sliderdir: slider directory
- :return: the configuration directory -any env var will
- override the relative path
- """
- localconf = os.path.join(sliderdir, CONF)
- return os.environ.get(SLIDER_CONF_DIR,localconf)
-
-def dirMustExist(dir):
- if not os.path.exists(dir):
- raise Exception("Directory does not exist: %s " % dir)
- return dir
-
-def read(pipe, line):
- """
- read a char, append to the listing if there is a char that is not \n
- :param pipe: pipe to read from
- :param line: line being built up
- :return: (the potentially updated line, flag indicating newline reached)
- """
-
- c = pipe.read(1)
- if c != "":
- o = c.decode('utf-8')
- if o != '\n':
- line += o
- return line, False
- else:
- return line, True
- else:
- return line, False
-
-
-
-def usage():
- print "Usage: slider <action> <arguments>"
- return 1
-
-
-def main():
- """
- Slider main method
- :return: exit code of the process
- """
- if len(sys.argv)==1 :
- return usage()
- print "stdout encoding: "+ sys.stdout.encoding
- args = sys.argv[1:]
- slider_home = sliderDir()
- libdir = dirMustExist(libDir(slider_home))
- confdir = dirMustExist(confDir(slider_home))
- default_jvm_opts = DEFAULT_JVM__OPTS % confdir
- slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
- # split the JVM opts by space
- jvm_opts_split = slider_jvm_opts.split()
- slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
- p = os.pathsep # path separator
- d = os.sep # dir separator
- slider_classpath = libdir + d + "*" + p \
- + confdir + p \
- + slider_classpath_extra
-
-
- print "slider_home = \"%s\"" % slider_home
- print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
- print "slider_classpath = \"%s\"" % slider_classpath
-
- #java = "/usr/bin/java"
- commandline = ["java", ]
- commandline.append("-classpath")
- commandline.append(slider_classpath)
- commandline.extend(jvm_opts_split)
- commandline.append(SLIDER_CLASSNAME)
- commandline.extend(args)
- print "ready to exec : %s" % commandline
- # docs warn of using PIPE on stderr
- exe = subprocess.Popen(commandline,
- stdin=None,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
- while exe.poll() is None:
- # process is running; grab output and echo every line
- outline, done = read(stdout, outline)
- if done:
- print outline
- outline = ""
- errline, done = read(stderr, errline)
- if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
- return exe.returncode
-
-
-
-if __name__ == '__main__':
- """
- Entry point
- """
- try:
- returncode = main()
- except Exception as e:
- print "Exception: %s " % e.message
- returncode = -1
-
- sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f05995e/slider-assembly/src/main/scripts/slider.sh
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.sh b/slider-assembly/src/main/scripts/slider.sh
new file mode 100755
index 0000000..caf275b
--- /dev/null
+++ b/slider-assembly/src/main/scripts/slider.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# this is the shell script to start Slider deploying an application
+# Usage: slider <action> <commands>
+
+# The env variable SLIDER_JVM_OPTS can be used to override
+# the default JVM opts
+
+function usage
+{
+ echo "Usage: slider <action> <arguments>"
+ echo ""
+}
+
+# Slider works out its own location
+this="${BASH_SOURCE-$0}"
+bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
+script="$(basename -- "$this")"
+
+# lib directory is one up; it is expected to contain
+# slider.jar and any other dependencies that are not in the
+# standard Hadoop classpath
+
+slider_home="${bindir}/.."
+slider_home=`cd -P "${slider_home}" && pwd -P`
+
+libdir="${slider_home}/lib"
+libdir=`cd -P "${libdir}" && pwd -P`
+
+
+confdir="${slider_home}/conf"
+
+# normalize the conf dir so it can be passed down
+confdir=`cd -P "${confdir}" && pwd -P`
+confdir=${SLIDER_CONF_DIR:-$confdir}
+
+
+slider_jvm_opts="-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=${confdir}"
+slider_jvm_opts=${SLIDER_JVM_OPTS:-$slider_jvm_opts}
+
+# allow for an extra classpath
+slider_classpath_extra=${SLIDER_CLASSPATH_EXTRA:-""}
+
+slider_classpath="${libdir}/*:${confdir}:${slider_classpath_extra}"
+
+launcher=org.apache.slider.Slider
+
+
+echo "slider_home = \"${slider_home}\""
+echo "slider_jvm_opts = \"${slider_jvm_opts}\""
+echo "classpath = \"${slider_classpath}\""
+export CLASSPATH="${slider_classpath}"
+echo ""
+
+echo "command is java ${slider_jvm_opts} --classpath \"${slider_classpath}\" ${launcher} $@"
+echo ""
+echo ""
+exec java ${slider_jvm_opts} ${launcher} $@
[27/50] [abbrv] git commit: SLIDER-153 add JVM args to CLI
Posted by st...@apache.org.
SLIDER-153 add JVM args to CLI
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8a8ffd94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8a8ffd94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8a8ffd94
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 8a8ffd94c9b9685df578ba493e6cb0148af0e5b1
Parents: 77f3e02
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 13:19:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 13:19:49 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8a8ffd94/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index 77c39aa..d48eca6 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -30,14 +30,12 @@ SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
SLIDER_CLASSNAME = "org.apache.slider.Slider"
DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
-"""Launches slider
-
-
"""
+Launches slider
+"""
-print os.environ['HOME']
def scriptDir():
@@ -107,6 +105,8 @@ def main():
confdir = dirMustExist(confDir(slider_home))
default_jvm_opts = DEFAULT_JVM__OPTS % confdir
slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
+ # split the JVM opts by space
+ jvm_opts_split = slider_jvm_opts.split()
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
p = os.pathsep # path separator
d = os.sep # dir separator
@@ -120,12 +120,11 @@ def main():
print "slider_classpath = \"%s\"" % slider_classpath
#java = "/usr/bin/java"
- java = "java"
- commandline = [java,
- "-classpath",
- slider_classpath,
- SLIDER_CLASSNAME]
- # commandline.append(slider_jvm_opts)
+ commandline = ["java", ]
+ commandline.append("-classpath")
+ commandline.append(slider_classpath)
+ commandline.extend(jvm_opts_split)
+ commandline.append(SLIDER_CLASSNAME)
commandline.extend(args)
print "ready to exec : %s" % commandline
# docs warn of using PIPE on stderr
[32/50] [abbrv] git commit: SLIDER-153 remove superflous check for
agent test enablement in TestAppsThroughAgent
Posted by st...@apache.org.
SLIDER-153 remove superflous check for agent test enablement in TestAppsThroughAgent
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bab88672
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bab88672
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bab88672
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: bab886723eca65bbe3a4090642b26714391df3ba
Parents: eba97da
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 18:10:02 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 18:10:02 2014 -0700
----------------------------------------------------------------------
.../apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bab88672/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
index e3422ee..793a323 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
@@ -43,10 +43,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
@Test
public void testCreateFlex() throws Throwable {
- if (!AGENTTESTS_ENABLED) {
- log.info "TESTS are not run."
- return
- }
+ assumeAgentTestsEnabled()
cleanup(APPLICATION_NAME)
SliderShell shell = slider(EXIT_SUCCESS,
[25/50] [abbrv] git commit: SLIDER-153: output streaming now working
Posted by st...@apache.org.
SLIDER-153: output streaming now working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6fc569de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6fc569de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6fc569de
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 6fc569de084dbba6a9701bba5595f557d1029484
Parents: 06816c4
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 13:01:37 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 13:01:37 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 60 +++++++++++++++++++------
1 file changed, 46 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6fc569de/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index cfad59a..1e55cc6 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -67,6 +67,26 @@ def dirMustExist(dir):
raise Exception("Directory does not exist: %s " % dir)
return dir
+def read(pipe, line):
+ """
+ read a char, append to the listing if there is a char that is not \n
+ :param pipe: pipe to read from
+ :param line: line being built up
+ :return: (the potentially updated line, flag indicating newline reached)
+ """
+
+ c = pipe.read(1)
+ if c != "":
+ o = c.decode('utf-8')
+ if o != '\n':
+ line += o
+ return line, False
+ else:
+ return line, True
+ else:
+ return line, False
+
+
def usage():
print "Usage: slider <action> <arguments>"
@@ -80,6 +100,7 @@ def main():
"""
if len(sys.argv)==1 :
return usage()
+ print "stdout encoding: "+ sys.stdout.encoding
args = sys.argv[1:]
slider_home = sliderDir()
libdir = dirMustExist(libDir(slider_home))
@@ -109,30 +130,41 @@ def main():
print "ready to exec : %s" % commandline
# docs warn of using PIPE on stderr
exe = subprocess.Popen(commandline,
- stdin=subprocess.PIPE,
+ stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
- exe.wait()
+ stdout = exe.stdout
+ stderr = exe.stderr
+ outline = ""
+ errline = ""
+ while exe.poll() is None:
+ # process is running; grab output and echo every line
+ outline, done = read(stdout, outline)
+ if done:
+ print outline
+ outline = ""
+ errline, done = read(stderr, errline)
+ if done:
+ print errline
+ errline = ""
+
+ # get tail
out, err = exe.communicate()
- print "stdout : ", out.decode()
- print "stderr : ", err.decode()
+ print outline + out.decode()
+ print errline + err.decode()
return exe.returncode
-
-
if __name__ == '__main__':
- print "slider python script"
+ """
+ Entry point
+ """
try:
- rv = main()
- if rv != 0:
- print "Failed with exit code = %d" % rv
- else:
- print "Success"
+ returncode = main()
except Exception as e:
print "Exception: %s " % e.message
- rv = -1
+ returncode = -1
- sys.exit(rv)
+ sys.exit(returncode)
[05/50] [abbrv] git commit: SLIDER-156 Utilize multiple log
directories when possible
Posted by st...@apache.org.
SLIDER-156 Utilize multiple log directories when possible
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/57a9c9f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/57a9c9f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/57a9c9f8
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 57a9c9f88d7bc41ee80320dd146244be5c3d420f
Parents: 45f5d79
Author: tedyu <yu...@gmail.com>
Authored: Sun Jun 22 17:52:20 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Sun Jun 22 17:52:20 2014 -0700
----------------------------------------------------------------------
.../slider/providers/hbase/HBaseProviderService.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/57a9c9f8/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 5405753..a578441 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -60,6 +60,7 @@ import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_PUBLISHER;
@@ -127,9 +128,16 @@ public class HBaseProviderService extends AbstractProviderService implements
// Set the environment
launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
- String logDir = providerUtils.getLogdir();
- int idx = logDir.indexOf(",");
- launcher.setEnv(HBASE_LOG_DIR, idx > 0 ? logDir.substring(0, idx) : logDir);
+ String logDirs = providerUtils.getLogdir();
+ String logDir;
+ int idx = logDirs.indexOf(",");
+ if (idx > 0) {
+ // randomly choose a log dir candidate
+ String[] segments = logDirs.split(",");
+ Random rand = new Random();
+ logDir = segments[rand.nextInt(segments.length)];
+ } else logDir = logDirs;
+ launcher.setEnv(HBASE_LOG_DIR, logDir);
launcher.setEnv(PROPAGATED_CONFDIR,
ProviderUtils.convertToAppRelativePath(
[13/50] [abbrv] git commit: SLIDER-153 more on execing scripts
Posted by st...@apache.org.
SLIDER-153 more on execing scripts
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/acdfe5bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/acdfe5bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/acdfe5bd
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: acdfe5bd6dfe59daca573a5e28271d9950467b16
Parents: 918e746
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 14:47:28 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 14:47:28 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/acdfe5bd/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index e846e64..2c1f4b3 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -89,11 +89,10 @@ def main():
slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
p = os.pathsep # path separator
d = os.sep # dir separator
- slider_classpath = '"' + \
- libdir + d + "*" + p \
+ slider_classpath = libdir + d + "*" + p \
+ confdir + p \
- + slider_classpath_extra \
- + '"'
+ + slider_classpath_extra
+
print "slider_home = \"%s\"" % slider_home
print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
@@ -111,7 +110,7 @@ def main():
return subprocess.call(commandline,
stdin=None,
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
shell=False)
[46/50] [abbrv] git commit: SLIDER-107. Work items for the release
0.40
Posted by st...@apache.org.
SLIDER-107. Work items for the release 0.40
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/736a9c7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/736a9c7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/736a9c7e
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 736a9c7e87aa4f31470bcf6016b004b9a83954b5
Parents: 590cbe0
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jun 26 17:29:58 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jun 26 20:13:44 2014 -0700
----------------------------------------------------------------------
LICENSE | 423 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
LICENSE.txt | 423 -------------------------------------------------------
NOTICE | 5 +
NOTICE.txt | 5 -
4 files changed, 428 insertions(+), 428 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/736a9c7e/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..b029173
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,423 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+
+APACHE SLIDER SUBCOMPONENTS:
+
+The Apache Slider project contains subcomponents with separate copyright
+notices and license terms. Your use of the source code for the these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+For Python Mock unit test framework,
+ at slider-agent/src/test/python/mock, the BSD license:
+
+ Copyright (c) 2003-2012, Michael Foord
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are
+ met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+For Kokki, resource management library,
+ which slider-agent/src/main/python/resource_management is derived from,
+ the BSD license:
+
+Copyright (c) 2009 Samuel Stauffer <sa...@descolada.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+ 3. Neither the name of Samuel Stauffer nor the names of its contributors may be used
+ to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+For jinja2, used by resource management library as a templating engine,
+ at slider-agent/src/main/python/jinja2, the BSD license:
+
+Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details.
+
+Some rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following
+ disclaimer in the documentation and/or other materials provided
+ with the distribution.
+
+ * The names of the contributors may not be used to endorse or
+ promote products derived from this software without specific
+ prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+For datatables,
+ at slider-core/src/main/resources/webapps/static/dt-1.9.4,
+ the BSD license:
+
+Copyright (c) 2008-2013, Allan Jardine All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Allan Jardine nor SpryMedia may be used to
+ endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY
+EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+
+For jquery,
+ at slider-core/src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js,
+ the MIT license:
+
+Copyright 2014 jQuery Foundation and other contributors
+http://jquery.com/
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For jquery-ui,
+ at slider-core/src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js
+ and slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/,
+ the MIT license:
+
+Copyright 2014 jQuery Foundation and other contributors,
+http://jqueryui.com/
+
+This software consists of voluntary contributions made by many
+individuals (AUTHORS.txt, http://jqueryui.com/about) For exact
+contribution history, see the revision history and logs, available
+at http://jquery-ui.googlecode.com/svn/
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+For jstree,
+ at slider-core/src/main/resources/webapps/static/jt/jquery.jstree.js,
+ the MIT license:
+
+Copyright (c) 2010 Ivan Bozhanov (vakata.com)
+http://jstree.com/
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/736a9c7e/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
deleted file mode 100644
index b029173..0000000
--- a/LICENSE.txt
+++ /dev/null
@@ -1,423 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-
-APACHE SLIDER SUBCOMPONENTS:
-
-The Apache Slider project contains subcomponents with separate copyright
-notices and license terms. Your use of the source code for the these
-subcomponents is subject to the terms and conditions of the following
-licenses.
-
-For Python Mock unit test framework,
- at slider-agent/src/test/python/mock, the BSD license:
-
- Copyright (c) 2003-2012, Michael Foord
- All rights reserved.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-For Kokki, resource management library,
- which slider-agent/src/main/python/resource_management is derived from,
- the BSD license:
-
-Copyright (c) 2009 Samuel Stauffer <sa...@descolada.com>
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
-
- 2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- 3. Neither the name of Samuel Stauffer nor the names of its contributors may be used
- to endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-For jinja2, used by resource management library as a templating engine,
- at slider-agent/src/main/python/jinja2, the BSD license:
-
-Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details.
-
-Some rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- * Redistributions in binary form must reproduce the above
- copyright notice, this list of conditions and the following
- disclaimer in the documentation and/or other materials provided
- with the distribution.
-
- * The names of the contributors may not be used to endorse or
- promote products derived from this software without specific
- prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-For datatables,
- at slider-core/src/main/resources/webapps/static/dt-1.9.4,
- the BSD license:
-
-Copyright (c) 2008-2013, Allan Jardine All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
- * Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in
- the documentation and/or other materials provided with the
- distribution.
- * Neither the name of Allan Jardine nor SpryMedia may be used to
- endorse or promote products derived from this software without
- specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS "AS IS" AND ANY
-EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
-THE POSSIBILITY OF SUCH DAMAGE.
-
-For jquery,
- at slider-core/src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js,
- the MIT license:
-
-Copyright 2014 jQuery Foundation and other contributors
-http://jquery.com/
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-For jquery-ui,
- at slider-core/src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js
- and slider-core/src/main/resources/webapps/static/jquery/themes-1.9.1/,
- the MIT license:
-
-Copyright 2014 jQuery Foundation and other contributors,
-http://jqueryui.com/
-
-This software consists of voluntary contributions made by many
-individuals (AUTHORS.txt, http://jqueryui.com/about) For exact
-contribution history, see the revision history and logs, available
-at http://jquery-ui.googlecode.com/svn/
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-For jstree,
- at slider-core/src/main/resources/webapps/static/jt/jquery.jstree.js,
- the MIT license:
-
-Copyright (c) 2010 Ivan Bozhanov (vakata.com)
-http://jstree.com/
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/736a9c7e/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..9b6f594
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,5 @@
+Apache Slider
+Copyright 2014 The Apache Software Foundation
+
+This product includes software developed at The Apache Software
+Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/736a9c7e/NOTICE.txt
----------------------------------------------------------------------
diff --git a/NOTICE.txt b/NOTICE.txt
deleted file mode 100644
index 86ac395..0000000
--- a/NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-Apache Slider
-Copyright 2007-2014 The Apache Software Foundation
-
-This product includes software developed at The Apache Software
-Foundation (http://www.apache.org/).
[10/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-153_add_slider_py_command
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-153_add_slider_py_command
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e519fbcc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e519fbcc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e519fbcc
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: e519fbcc645894541bdf93d1fef9ceba36ded847
Parents: 19d313d 537d87c
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 13:33:20 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 13:33:20 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-v0_96/README.txt | 33 --
app-packages/hbase-v0_96/appConfig.json | 67 ----
.../hbase-v0_96/configuration/global.xml | 160 --------
.../hbase-v0_96/configuration/hbase-log4j.xml | 142 --------
.../hbase-v0_96/configuration/hbase-policy.xml | 53 ---
.../hbase-v0_96/configuration/hbase-site.xml | 365 -------------------
app-packages/hbase-v0_96/ganglia_metrics.json | 38 --
app-packages/hbase-v0_96/jmx_metrics.json | 56 ---
app-packages/hbase-v0_96/metainfo.xml | 109 ------
.../hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE | 16 -
.../hbase-v0_96/package/scripts/__init__.py | 19 -
.../hbase-v0_96/package/scripts/functions.py | 40 --
.../hbase-v0_96/package/scripts/hbase.py | 125 -------
.../hbase-v0_96/package/scripts/hbase_client.py | 43 ---
.../hbase-v0_96/package/scripts/hbase_master.py | 63 ----
.../package/scripts/hbase_regionserver.py | 66 ----
.../package/scripts/hbase_service.py | 45 ---
.../hbase-v0_96/package/scripts/params.py | 109 ------
.../package/scripts/status_params.py | 26 --
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 62 ----
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 62 ----
.../package/templates/hbase-env.sh.j2 | 81 ----
.../package/templates/hbase_client_jaas.conf.j2 | 22 --
.../package/templates/hbase_master_jaas.conf.j2 | 25 --
.../templates/hbase_regionserver_jaas.conf.j2 | 25 --
.../package/templates/regionservers.j2 | 20 -
app-packages/hbase-v0_96/resources.json | 19 -
app-packages/hbase/README.txt | 8 +-
slider-agent/conf/agent.ini | 2 +
.../src/main/python/agent/AgentConfig.py | 21 ++
slider-agent/src/main/python/agent/Constants.py | 3 +
.../src/main/python/agent/Controller.py | 23 +-
slider-agent/src/main/python/agent/main.py | 4 +
.../src/test/python/agent/TestController.py | 30 ++
slider-agent/src/test/python/agent/TestMain.py | 5 +-
.../providers/AbstractProviderService.java | 15 +-
.../slider/providers/ProviderService.java | 5 +-
.../slider/providers/agent/AgentKeys.java | 3 +
.../providers/agent/AgentLaunchParameter.java | 130 +++++++
.../providers/agent/AgentProviderService.java | 179 +++++++--
.../slider/providers/agent/AgentRoles.java | 18 +-
.../apache/slider/providers/agent/Command.java | 13 +-
.../slider/providers/agent/CommandResult.java | 16 +-
.../providers/agent/ComponentInstanceState.java | 24 ++
.../slider/providers/agent/ContainerState.java | 41 +++
.../providers/agent/HeartbeatMonitor.java | 116 ++++++
.../server/appmaster/AMViewForProviders.java | 27 ++
.../server/appmaster/SliderAppMaster.java | 32 +-
.../slider/server/appmaster/state/AppState.java | 20 +
.../services/workflow/ForkedProcessService.java | 25 +-
.../services/workflow/LongLivedProcess.java | 13 +
.../test_command_log/appConfig_fast_no_reg.json | 29 ++
.../test_command_log/appConfig_no_hb.json | 29 ++
.../model/mock/MockProviderService.groovy | 4 +-
.../agent/TestAgentLaunchParameter.java | 76 ++++
.../providers/agent/TestHeartbeatMonitor.java | 136 +++++++
.../publisher/TestAgentProviderService.java | 5 +-
.../TestWorkflowForkedProcessService.java | 3 +-
.../lifecycle/AgentCommandTestBase.groovy | 100 +++--
.../lifecycle/TestAgentClusterLifecycle.groovy | 2 +-
.../funtest/lifecycle/TestAgentFailures.groovy | 103 ++++++
.../funtest/lifecycle/TestAgentFailures2.groovy | 103 ++++++
.../lifecycle/TestAppsThroughAgent.groovy | 90 ++---
.../providers/hbase/HBaseProviderService.java | 13 +-
.../slider_specs/creating_app_definitions.md | 80 ++--
65 files changed, 1349 insertions(+), 2088 deletions(-)
----------------------------------------------------------------------
[11/50] [abbrv] git commit: SLIDER-153 stub main()
Posted by st...@apache.org.
SLIDER-153 stub main()
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d6131ca4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d6131ca4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d6131ca4
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: d6131ca426bee6e8aa9819a377bb82345fc97b56
Parents: e519fbc
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 13:41:18 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 13:41:18 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider.py | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d6131ca4/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index f154210..d5891ed 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -15,8 +15,14 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import sys
+import subprocess
# Slider main method
-
+def main():
+
+
if __name__ == '__main__':
- print "slider python script"
\ No newline at end of file
+ print "slider python script"
+ rv = main()
+ sys.exit(rv)
[12/50] [abbrv] git commit: SLIDER-153 building up exec
Posted by st...@apache.org.
SLIDER-153 building up exec
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/918e746d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/918e746d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/918e746d
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 918e746d83c4ad0f4a25bba39050942f5689599a
Parents: d6131ca
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 14:40:34 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 14:40:34 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 1 -
slider-assembly/src/main/scripts/slider.py | 109 +++++++++++++++++++++++-
2 files changed, 106 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/918e746d/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
index e9522cf..caf275b 100755
--- a/slider-assembly/src/main/scripts/slider
+++ b/slider-assembly/src/main/scripts/slider
@@ -32,7 +32,6 @@ function usage
this="${BASH_SOURCE-$0}"
bindir=$(cd -P -- "$(dirname -- "$this")" && pwd -P)
script="$(basename -- "$this")"
-this="$bin/$script"
# lib directory is one up; it is expected to contain
# slider.jar and any other dependencies that are not in the
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/918e746d/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
index d5891ed..e846e64 100644
--- a/slider-assembly/src/main/scripts/slider.py
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -16,13 +16,116 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
+import os
import subprocess
-# Slider main method
+CONF = "conf"
+
+LIB = "lib"
+
+SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
+SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
+SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
+
+SLIDER_CLASSNAME = "org.apache.slider.Slider"
+DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
+
+"""Launches slider
+
+
+"""
+
+
+
+print os.environ['HOME']
+
+
+def scriptDir():
+ """
+ get the script path
+ """
+ return os.path.dirname(os.path.realpath(__file__))
+
+def sliderDir():
+ return os.path.dirname(scriptDir())
+
+def libDir(sliderdir) :
+ return os.path.join(sliderdir, LIB)
+
+def confDir(sliderdir):
+ """
+ determine the active configuration directory
+ :param sliderdir: slider directory
+ :return: the configuration directory -any env var will
+ override the relative path
+ """
+ localconf = os.path.join(sliderdir, CONF)
+ return os.environ.get(SLIDER_CONF_DIR,localconf)
+
+def dirMustExist(dir):
+ if not os.path.exists(dir):
+ raise Exception("Directory does not exist: %s " % dir)
+ return dir
+
+
+def usage():
+ print "Usage: slider <action> <arguments>"
+ return 1
+
+
def main():
+ """
+ Slider main method
+ :return: exit code of the process
+ """
+ if len(sys.argv)==1 :
+ return usage()
+ args = sys.argv[1:]
+ slider_home = sliderDir()
+ libdir = dirMustExist(libDir(slider_home))
+ confdir = dirMustExist(confDir(slider_home))
+ default_jvm_opts = DEFAULT_JVM__OPTS % confdir
+ slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
+ slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
+ p = os.pathsep # path separator
+ d = os.sep # dir separator
+ slider_classpath = '"' + \
+ libdir + d + "*" + p \
+ + confdir + p \
+ + slider_classpath_extra \
+ + '"'
+
+ print "slider_home = \"%s\"" % slider_home
+ print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
+ print "slider_classpath = \"%s\"" % slider_classpath
+ commandline = ["java",]
+ # commandline.append(slider_jvm_opts)
+ commandline.append("-classpath")
+ commandline.append(slider_classpath)
+ commandline.append(SLIDER_CLASSNAME)
+ commandline.extend(args)
+ print "ready to exec : %s" % commandline
+ # docs warn of using PIPE on stderr
+ return subprocess.call(commandline,
+ stdin=None,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=False)
+
+
+
+
+
if __name__ == '__main__':
- print "slider python script"
+ print "slider python script"
+ try:
rv = main()
- sys.exit(rv)
+ if rv != 0:
+ print "exit code = %d" % rv
+ except Exception as e:
+ print "Exception: %s " % e.message
+ rv = -1
+
+ sys.exit(rv)
[35/50] [abbrv] git commit: SLIDER-153 revert rename -> slider.py for
python
Posted by st...@apache.org.
SLIDER-153 revert rename -> slider.py for python
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1c5f303f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1c5f303f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1c5f303f
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 1c5f303f31693b89c1d767043655b16922ae0b57
Parents: 6f05995
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jun 25 11:21:29 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jun 25 11:21:29 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 169 ------------------------
slider-assembly/src/main/scripts/slider.py | 169 ++++++++++++++++++++++++
2 files changed, 169 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1c5f303f/slider-assembly/src/main/scripts/slider
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider b/slider-assembly/src/main/scripts/slider
deleted file mode 100644
index d48eca6..0000000
--- a/slider-assembly/src/main/scripts/slider
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import sys
-import os
-import subprocess
-
-CONF = "conf"
-
-LIB = "lib"
-
-SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
-SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
-SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
-
-SLIDER_CLASSNAME = "org.apache.slider.Slider"
-DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
-
-"""
-Launches slider
-
-
-"""
-
-
-
-def scriptDir():
- """
- get the script path
- """
- return os.path.dirname(os.path.realpath(__file__))
-
-def sliderDir():
- return os.path.dirname(scriptDir())
-
-def libDir(sliderdir) :
- return os.path.join(sliderdir, LIB)
-
-def confDir(sliderdir):
- """
- determine the active configuration directory
- :param sliderdir: slider directory
- :return: the configuration directory -any env var will
- override the relative path
- """
- localconf = os.path.join(sliderdir, CONF)
- return os.environ.get(SLIDER_CONF_DIR,localconf)
-
-def dirMustExist(dir):
- if not os.path.exists(dir):
- raise Exception("Directory does not exist: %s " % dir)
- return dir
-
-def read(pipe, line):
- """
- read a char, append to the listing if there is a char that is not \n
- :param pipe: pipe to read from
- :param line: line being built up
- :return: (the potentially updated line, flag indicating newline reached)
- """
-
- c = pipe.read(1)
- if c != "":
- o = c.decode('utf-8')
- if o != '\n':
- line += o
- return line, False
- else:
- return line, True
- else:
- return line, False
-
-
-
-def usage():
- print "Usage: slider <action> <arguments>"
- return 1
-
-
-def main():
- """
- Slider main method
- :return: exit code of the process
- """
- if len(sys.argv)==1 :
- return usage()
- print "stdout encoding: "+ sys.stdout.encoding
- args = sys.argv[1:]
- slider_home = sliderDir()
- libdir = dirMustExist(libDir(slider_home))
- confdir = dirMustExist(confDir(slider_home))
- default_jvm_opts = DEFAULT_JVM__OPTS % confdir
- slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
- # split the JVM opts by space
- jvm_opts_split = slider_jvm_opts.split()
- slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
- p = os.pathsep # path separator
- d = os.sep # dir separator
- slider_classpath = libdir + d + "*" + p \
- + confdir + p \
- + slider_classpath_extra
-
-
- print "slider_home = \"%s\"" % slider_home
- print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
- print "slider_classpath = \"%s\"" % slider_classpath
-
- #java = "/usr/bin/java"
- commandline = ["java", ]
- commandline.append("-classpath")
- commandline.append(slider_classpath)
- commandline.extend(jvm_opts_split)
- commandline.append(SLIDER_CLASSNAME)
- commandline.extend(args)
- print "ready to exec : %s" % commandline
- # docs warn of using PIPE on stderr
- exe = subprocess.Popen(commandline,
- stdin=None,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- shell=False)
- stdout = exe.stdout
- stderr = exe.stderr
- outline = ""
- errline = ""
- while exe.poll() is None:
- # process is running; grab output and echo every line
- outline, done = read(stdout, outline)
- if done:
- print outline
- outline = ""
- errline, done = read(stderr, errline)
- if done:
- print errline
- errline = ""
-
- # get tail
- out, err = exe.communicate()
- print outline + out.decode()
- print errline + err.decode()
- return exe.returncode
-
-
-
-if __name__ == '__main__':
- """
- Entry point
- """
- try:
- returncode = main()
- except Exception as e:
- print "Exception: %s " % e.message
- returncode = -1
-
- sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1c5f303f/slider-assembly/src/main/scripts/slider.py
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/scripts/slider.py b/slider-assembly/src/main/scripts/slider.py
new file mode 100644
index 0000000..d48eca6
--- /dev/null
+++ b/slider-assembly/src/main/scripts/slider.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+import os
+import subprocess
+
+CONF = "conf"
+
+LIB = "lib"
+
+SLIDER_CONF_DIR = "SLIDER_CONF_DIR"
+SLIDER_JVM_OPTS = "SLIDER_JVM_OPTS"
+SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
+
+SLIDER_CLASSNAME = "org.apache.slider.Slider"
+DEFAULT_JVM__OPTS = "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Djava.confdir=%s"
+
+"""
+Launches slider
+
+
+"""
+
+
+
+def scriptDir():
+ """
+ get the script path
+ """
+ return os.path.dirname(os.path.realpath(__file__))
+
+def sliderDir():
+ return os.path.dirname(scriptDir())
+
+def libDir(sliderdir) :
+ return os.path.join(sliderdir, LIB)
+
+def confDir(sliderdir):
+ """
+ determine the active configuration directory
+ :param sliderdir: slider directory
+ :return: the configuration directory -any env var will
+ override the relative path
+ """
+ localconf = os.path.join(sliderdir, CONF)
+ return os.environ.get(SLIDER_CONF_DIR,localconf)
+
+def dirMustExist(dir):
+ if not os.path.exists(dir):
+ raise Exception("Directory does not exist: %s " % dir)
+ return dir
+
+def read(pipe, line):
+ """
+ read a char, append to the listing if there is a char that is not \n
+ :param pipe: pipe to read from
+ :param line: line being built up
+ :return: (the potentially updated line, flag indicating newline reached)
+ """
+
+ c = pipe.read(1)
+ if c != "":
+ o = c.decode('utf-8')
+ if o != '\n':
+ line += o
+ return line, False
+ else:
+ return line, True
+ else:
+ return line, False
+
+
+
+def usage():
+ print "Usage: slider <action> <arguments>"
+ return 1
+
+
+def main():
+ """
+ Slider main method
+ :return: exit code of the process
+ """
+ if len(sys.argv)==1 :
+ return usage()
+ print "stdout encoding: "+ sys.stdout.encoding
+ args = sys.argv[1:]
+ slider_home = sliderDir()
+ libdir = dirMustExist(libDir(slider_home))
+ confdir = dirMustExist(confDir(slider_home))
+ default_jvm_opts = DEFAULT_JVM__OPTS % confdir
+ slider_jvm_opts = os.environ.get(SLIDER_JVM_OPTS, default_jvm_opts)
+ # split the JVM opts by space
+ jvm_opts_split = slider_jvm_opts.split()
+ slider_classpath_extra = os.environ.get(SLIDER_CLASSPATH_EXTRA, "")
+ p = os.pathsep # path separator
+ d = os.sep # dir separator
+ slider_classpath = libdir + d + "*" + p \
+ + confdir + p \
+ + slider_classpath_extra
+
+
+ print "slider_home = \"%s\"" % slider_home
+ print "slider_jvm_opts = \"%s\"" % slider_jvm_opts
+ print "slider_classpath = \"%s\"" % slider_classpath
+
+ #java = "/usr/bin/java"
+ commandline = ["java", ]
+ commandline.append("-classpath")
+ commandline.append(slider_classpath)
+ commandline.extend(jvm_opts_split)
+ commandline.append(SLIDER_CLASSNAME)
+ commandline.extend(args)
+ print "ready to exec : %s" % commandline
+ # docs warn of using PIPE on stderr
+ exe = subprocess.Popen(commandline,
+ stdin=None,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ shell=False)
+ stdout = exe.stdout
+ stderr = exe.stderr
+ outline = ""
+ errline = ""
+ while exe.poll() is None:
+ # process is running; grab output and echo every line
+ outline, done = read(stdout, outline)
+ if done:
+ print outline
+ outline = ""
+ errline, done = read(stderr, errline)
+ if done:
+ print errline
+ errline = ""
+
+ # get tail
+ out, err = exe.communicate()
+ print outline + out.decode()
+ print errline + err.decode()
+ return exe.returncode
+
+
+
+if __name__ == '__main__':
+ """
+ Entry point
+ """
+ try:
+ returncode = main()
+ except Exception as e:
+ print "Exception: %s " % e.message
+ returncode = -1
+
+ sys.exit(returncode)
[37/50] [abbrv] git commit: Merge branch
'feature/SLIDER-153_add_slider_py_command' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-153_add_slider_py_command' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7089fa0f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7089fa0f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7089fa0f
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 7089fa0fbd6d1cb8c92eb1a360cdd82429321bdf
Parents: 1866733 150b589
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jun 25 11:22:59 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jun 25 11:22:59 2014 -0700
----------------------------------------------------------------------
slider-assembly/src/main/scripts/slider | 1 -
slider-assembly/src/main/scripts/slider.py | 169 ++++++++++++++++++++++++
2 files changed, 169 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[30/50] [abbrv] git commit: SLIDER-160: agent functional tests upload
binaries/ini file if there is a difference
Posted by st...@apache.org.
SLIDER-160: agent functional tests upload binaries/ini file if there is a difference
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5464584d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5464584d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5464584d
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 5464584d784cf57348bf22100cee2581d5a887c0
Parents: 8327e52
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 24 17:40:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 24 17:40:49 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 41 +++++++++++++++---
.../funtest/framework/CommandTestBase.groovy | 9 ++--
.../funtest/framework/FileUploader.groovy | 45 ++++++++++++++------
.../funtest/framework/FuntestProperties.groovy | 12 ++++--
.../lifecycle/AgentCommandTestBase.groovy | 39 ++++++-----------
.../lifecycle/TestClusterBuildDestroy.groovy | 2 +-
6 files changed, 94 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
index b1c29c0..5f10c0e 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -20,22 +20,51 @@ package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.fs.permission.FsPermission
import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
-class AgentUploads {
+class AgentUploads implements FuntestProperties {
final Configuration conf
+ private final FileUploader uploader
+ private final HadoopFS clusterFS
+ private final Path homeDir
+
AgentUploads(Configuration conf) {
this.conf = conf
+ uploader = new FileUploader(conf, UserGroupInformation.currentUser)
+ clusterFS = uploader.fileSystem
+ homeDir = clusterFS.homeDirectory
}
- def execUploadSequence() {
- UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn")
+ /**
+ * Upload agent-related files
+ * @param tarballDir
+ * @param force
+ * @return
+ */
+ def uploadAgentFiles(File tarballDir, boolean force) {
+ def localAgentTar = new File(tarballDir, AGENT_SLIDER_GZ_IN_SLIDER_TAR)
+ assert localAgentTar.exists()
+
+ def agentTarballPath = new Path(
+ homeDir,
+ AGENT_TAR_FILENAME)
+
+ // Upload the agent tarball
+ uploader.copyIfOutOfDate(localAgentTar, agentTarballPath, force)
+
+ File localAgentIni = new File(tarballDir, AGENT_INI_IN_SLIDER_TAR)
+ // Upload the agent.ini
+ def agentIniPath = new Path(homeDir, AGENT_INI)
+ uploader.copyIfOutOfDate(localAgentIni, agentIniPath, force)
+
+ return [agentTarballPath, agentIniPath]
+
+
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index d2accbb..61ae804 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -50,17 +50,16 @@ abstract class CommandTestBase extends SliderTestUtils {
LoggerFactory.getLogger(CommandTestBase.class);
public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP)
- public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP)
- public static final File SLIDER_BIN_DIRECTORY = new File(
- SLIDER_BIN_DIR).canonicalFile
+ public static final String SLIDER_TAR_DIR = sysprop(SLIDER_BIN_DIR_PROP)
+ public static final File SLIDER_TAR_DIRECTORY = new File(
+ SLIDER_TAR_DIR).canonicalFile
public static final File SLIDER_SCRIPT = new File(
- SLIDER_BIN_DIRECTORY,
+ SLIDER_TAR_DIRECTORY,
BIN_SLIDER).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(
SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
CLIENT_CONFIG_FILENAME).canonicalFile
-
public static final YarnConfiguration SLIDER_CONFIG
public static final int THAW_WAIT_TIME
public static final int FREEZE_WAIT_TIME
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
index dd42473..262693c 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -23,8 +23,9 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.AccessControlException
import org.apache.hadoop.security.UserGroupInformation
-
+import org.apache.hadoop.fs.FileSystem as HadoopFS
@Slf4j
class FileUploader {
final Configuration conf
@@ -38,39 +39,59 @@ class FileUploader {
/**
* Copy if the file is considered out of date
* @param src
- * @param dest
+ * @param destPath
* @param force
* @return
*/
- public boolean copyIfOutOfDate(File src, Path dest, boolean force) {
+ public boolean copyIfOutOfDate(File src, Path destPath, boolean force) {
def srcLen = src.length()
- def fs = getFileSystem(user, dest.toUri())
+ def fs = getFileSystem(destPath)
boolean toCopy = force
if (!toCopy) {
try {
- def status = fs.getFileStatus(dest)
+ def status = fs.getFileStatus(destPath)
toCopy = status.len != srcLen
} catch (FileNotFoundException fnfe) {
toCopy = true;
}
}
if (toCopy) {
- log.info("Copying $src to $dest")
- fs.mkdirs(dest, FsPermission.dirDefault)
- return FileUtil.copy(src, fs, dest, false, conf)
+ log.info("Copying $src to $destPath")
+ try {
+ fs.delete(destPath, true)
+ fs.mkdirs(destPath.getParent(), FsPermission.dirDefault)
+ return FileUtil.copy(src, fs, destPath, false, conf)
+ } catch (AccessControlException ace) {
+ log.error("No write access to test user home directory. " +
+ "Ensure home directory exists and has correct permissions." +
+ ace, ace)
+ throw ace
+ }
} else {
- log.debug("Skipping copy as the destination $dest considered up to date")
+ log.debug("Skipping copy as the destination $destPath considered up to date")
return false;
}
+ }
+ public HadoopFS getFileSystem(Path dest) {
+ getFileSystem(user, dest)
+ }
+
+ public HadoopFS getFileSystem() {
+ getFileSystem(user, HadoopFS.getDefaultUri(conf))
}
- public static def getFileSystem(
+ public def getFileSystem(
+ UserGroupInformation user, final Path path) {
+ return getFileSystem(user, path.toUri())
+
+ }
+ public def getFileSystem(
UserGroupInformation user, final URI uri) {
+
SudoClosure.sudo(user) {
- org.apache.hadoop.fs.FileSystem.get(uri, conf);
+ HadoopFS.get(uri, conf);
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 8cbc098..9b63c22 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -57,8 +57,12 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting {
String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA"
String SCRIPT_NAME = "slider"
- static final String KEY_TEST_CONF_XML = "slider.test.conf.xml"
- static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
- static final String BIN_SLIDER = "bin/slider"
- static final String AGENT_SLIDER_GZ = "agent/slider-agent.tar.gz"
+ String KEY_TEST_CONF_XML = "slider.test.conf.xml"
+ String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
+ String BIN_SLIDER = "bin/slider"
+ String AGENT_INI = "agent.ini"
+ String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI
+
+ String AGENT_TAR_FILENAME = "slider-agent.tar.gz"
+ String AGENT_SLIDER_GZ_IN_SLIDER_TAR = "agent/" + AGENT_TAR_FILENAME
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
index 6333fad..c3d6bd6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
@@ -20,10 +20,10 @@ package org.apache.slider.funtest.lifecycle
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.security.AccessControlException
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentUploads
import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -53,37 +53,29 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
static {
AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
- LOCAL_SLIDER_AGENT_TARGZ = new File(
- SLIDER_BIN_DIRECTORY,
- AGENT_SLIDER_GZ).canonicalFile
LOCAL_AGENT_CONF = new File(AGENT_CONF).canonicalFile
}
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ public static void assumeAgentTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(AGENTTESTS_ENABLED, "Agent tests disabled")
+ }
+
@BeforeClass
public static void setupAgent() {
assumeAgentTestsEnabled()
- try {
- // Upload the agent tarball
- assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
- agentTarballPath = new Path(clusterFS.homeDirectory, "slider-agent.tar.gz")
- Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
- clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath)
-
- // Upload the agent.ini
- assume(LOCAL_AGENT_CONF.exists(), "Agent config not found at $LOCAL_AGENT_CONF")
- agtIniPath = new Path(clusterFS.homeDirectory, "agent.ini")
- Path localAgtIni = new Path(LOCAL_AGENT_CONF.toURI());
- clusterFS.copyFromLocalFile(false, true, localAgtIni, agtIniPath)
- } catch (AccessControlException ace) {
- log.info "No write access to test user home directory. " +
- "Ensure home directory exists and has correct permissions." + ace.getMessage()
- fail("Ensure home directory exists and has correct permissions for test user.")
- }
}
+
+ @Before
+ public void uploadAgentTarball() {
+ def agentUploads = new AgentUploads(SLIDER_CONFIG)
+ (agentTarballPath, agtIniPath) =
+ agentUploads.uploadAgentFiles(SLIDER_TAR_DIRECTORY, false)
+ }
@Before
@@ -105,11 +97,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath)
}
- public static void assumeAgentTestsEnabled() {
- assumeFunctionalTestsEnabled()
- assume(AGENTTESTS_ENABLED, "Agent tests disabled")
- }
-
public static void logShell(SliderShell shell) {
for (String str in shell.out) {
log.info str
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
index 3db6c5b..1907a2c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
@@ -42,7 +42,7 @@ public class TestClusterBuildDestroy extends AgentCommandTestBase
@BeforeClass
public static void prepareCluster() {
- assumeFunctionalTestsEnabled();
+
setupCluster(CLUSTER)
}
[02/50] [abbrv] SLIDER-126. Slider-Agent and Agent Provider need to
handle component instance install/start failure gracefully
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3aca57d2/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
index aef9aa1..e3422ee 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
@@ -31,14 +31,14 @@ import org.junit.Test
@CompileStatic
@Slf4j
public class TestAppsThroughAgent extends AgentCommandTestBase
- implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
- private static String APPLICATION_NAME = "agenttst"
+ private static String APPLICATION_NAME = "happy-path-with-flex"
@After
public void destroyCluster() {
- cleanup()
+ cleanup(APPLICATION_NAME)
}
@Test
@@ -48,69 +48,39 @@ public class TestAppsThroughAgent extends AgentCommandTestBase
return
}
- cleanup()
- try {
- SliderShell shell = slider(EXIT_SUCCESS,
- [
- ACTION_CREATE, APPLICATION_NAME,
- ARG_IMAGE, agentTarballPath.toString(),
- ARG_TEMPLATE, APP_TEMPLATE,
- ARG_RESOURCES, APP_RESOURCE
- ])
+ cleanup(APPLICATION_NAME)
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_IMAGE, agentTarballPath.toString(),
+ ARG_TEMPLATE, APP_TEMPLATE,
+ ARG_RESOURCES, APP_RESOURCE
+ ])
- logShell(shell)
+ logShell(shell)
- ensureApplicationIsUp(APPLICATION_NAME)
+ ensureApplicationIsUp(APPLICATION_NAME)
- //flex
- slider(EXIT_SUCCESS,
- [
- ACTION_FLEX,
- APPLICATION_NAME,
- ARG_COMPONENT,
- COMMAND_LOGGER,
- "2"])
+ //flex
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_FLEX,
+ APPLICATION_NAME,
+ ARG_COMPONENT,
+ COMMAND_LOGGER,
+ "2"])
- // sleep till the new instance starts
- sleep(1000 * 10)
+ // sleep till the new instance starts
+ sleep(1000 * 10)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_STATUS,
- APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
- assertComponentCount(COMMAND_LOGGER, 2, shell)
+ assertComponentCount(COMMAND_LOGGER, 2, shell)
- shell = slider(EXIT_SUCCESS,
- [
- ACTION_LIST,
- APPLICATION_NAME])
-
- assert isAppRunning("RUNNING", shell), 'App is not running.'
-
- assertSuccess(shell)
- } finally {
- cleanup()
- }
- }
-
-
- public void cleanup() throws Throwable {
- log.info "Cleaning app instance, if exists, by name " + APPLICATION_NAME
- teardown(APPLICATION_NAME)
-
- // sleep till the instance is frozen
- sleep(1000 * 3)
-
- SliderShell shell = slider([
- ACTION_DESTROY,
- APPLICATION_NAME])
-
- if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
- logShell(shell)
- assert fail("Old cluster either should not exist or should get destroyed.")
- }
+ assertSuccess(shell)
+ assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
}
-
-
}
[18/50] [abbrv] git commit: SLIDER-163 Change recommended app_log_dir
value
Posted by st...@apache.org.
SLIDER-163 Change recommended app_log_dir value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/83e4e473
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/83e4e473
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/83e4e473
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 83e4e47385d41f4c2258d4c510bd7c1a0ebf2354
Parents: 153f5e9
Author: tedyu <yu...@gmail.com>
Authored: Tue Jun 24 11:07:39 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue Jun 24 11:07:39 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/83e4e473/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 2b7ec52..6225129 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -9,7 +9,7 @@
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/hbase-${hbase.version}-hadoop2-bin.tar.gz",
"site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_log_dir": "app/log",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}-hadoop2",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
[50/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/81a85495
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/81a85495
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/81a85495
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 81a85495de9dcbf6aed1a37594d0e29b95a180d5
Parents: 6f5c5fa 0ad4fb4
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 30 16:16:30 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 30 16:16:30 2014 +0100
----------------------------------------------------------------------
.gitignore | 10 +-
LICENSE | 423 ++++++++
LICENSE.txt | 423 --------
NOTICE | 5 +
NOTICE.txt | 5 -
app-packages/accumulo-v1_5/README.txt | 33 -
app-packages/accumulo-v1_5/appConfig.json | 62 --
.../configuration/accumulo-site.xml | 111 --
.../accumulo-v1_5/configuration/global.xml | 94 --
app-packages/accumulo-v1_5/jmx_metrics.json | 41 -
app-packages/accumulo-v1_5/metainfo.xml | 145 ---
.../files/accumulo-1.5.1-bin.tar.gz.REPLACE | 14 -
.../package/files/accumulo-metrics.xml | 60 --
.../accumulo-v1_5/package/files/auditLog.xml | 41 -
app-packages/accumulo-v1_5/package/files/gc | 16 -
.../package/files/generic_logger.xml | 83 --
.../package/files/log4j.properties | 41 -
.../accumulo-v1_5/package/files/masters | 16 -
.../accumulo-v1_5/package/files/monitor | 16 -
.../package/files/monitor_logger.xml | 64 --
app-packages/accumulo-v1_5/package/files/slaves | 16 -
.../accumulo-v1_5/package/files/tracers | 16 -
.../accumulo-v1_5/package/scripts/__init__.py | 19 -
.../package/scripts/accumulo_client.py | 43 -
.../package/scripts/accumulo_configuration.py | 134 ---
.../package/scripts/accumulo_gc.py | 24 -
.../package/scripts/accumulo_master.py | 24 -
.../package/scripts/accumulo_monitor.py | 24 -
.../package/scripts/accumulo_script.py | 110 --
.../package/scripts/accumulo_service.py | 52 -
.../package/scripts/accumulo_tracer.py | 24 -
.../package/scripts/accumulo_tserver.py | 24 -
.../accumulo-v1_5/package/scripts/params.py | 72 --
.../package/scripts/status_params.py | 26 -
.../package/templates/accumulo-env.sh.j2 | 42 -
app-packages/accumulo-v1_5/resources.json | 31 -
app-packages/accumulo/LICENSE.txt | 261 +++++
app-packages/accumulo/NOTICE.txt | 8 +
app-packages/accumulo/README.txt | 47 +
app-packages/accumulo/appConfig.json | 62 ++
.../accumulo/configuration/accumulo-site.xml | 111 ++
app-packages/accumulo/configuration/global.xml | 94 ++
app-packages/accumulo/jmx_metrics.json | 41 +
app-packages/accumulo/metainfo.xml | 147 +++
.../accumulo/package/files/accumulo-metrics.xml | 60 ++
.../accumulo/package/files/auditLog.xml | 41 +
app-packages/accumulo/package/files/gc | 16 +
.../accumulo/package/files/generic_logger.xml | 83 ++
.../accumulo/package/files/log4j.properties | 41 +
app-packages/accumulo/package/files/masters | 16 +
app-packages/accumulo/package/files/monitor | 16 +
.../accumulo/package/files/monitor_logger.xml | 64 ++
app-packages/accumulo/package/files/slaves | 16 +
app-packages/accumulo/package/files/tracers | 16 +
.../accumulo/package/scripts/__init__.py | 19 +
.../accumulo/package/scripts/accumulo_client.py | 43 +
.../package/scripts/accumulo_configuration.py | 135 +++
.../accumulo/package/scripts/accumulo_gc.py | 24 +
.../accumulo/package/scripts/accumulo_master.py | 24 +
.../package/scripts/accumulo_monitor.py | 24 +
.../accumulo/package/scripts/accumulo_script.py | 110 ++
.../package/scripts/accumulo_service.py | 52 +
.../accumulo/package/scripts/accumulo_tracer.py | 24 +
.../package/scripts/accumulo_tserver.py | 24 +
app-packages/accumulo/package/scripts/params.py | 76 ++
.../accumulo/package/scripts/status_params.py | 26 +
.../package/templates/accumulo-env.sh.j2 | 42 +
app-packages/accumulo/pom.xml | 93 ++
app-packages/accumulo/resources.json | 31 +
app-packages/accumulo/src/assembly/accumulo.xml | 71 ++
app-packages/hbase-v0_96/README.txt | 33 -
app-packages/hbase-v0_96/appConfig.json | 67 --
.../hbase-v0_96/configuration/global.xml | 160 ---
.../hbase-v0_96/configuration/hbase-log4j.xml | 142 ---
.../hbase-v0_96/configuration/hbase-policy.xml | 53 -
.../hbase-v0_96/configuration/hbase-site.xml | 365 -------
app-packages/hbase-v0_96/ganglia_metrics.json | 38 -
app-packages/hbase-v0_96/jmx_metrics.json | 56 -
app-packages/hbase-v0_96/metainfo.xml | 109 --
.../hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE | 16 -
.../hbase-v0_96/package/scripts/__init__.py | 19 -
.../hbase-v0_96/package/scripts/functions.py | 40 -
.../hbase-v0_96/package/scripts/hbase.py | 125 ---
.../hbase-v0_96/package/scripts/hbase_client.py | 43 -
.../hbase-v0_96/package/scripts/hbase_master.py | 63 --
.../package/scripts/hbase_regionserver.py | 66 --
.../package/scripts/hbase_service.py | 45 -
.../hbase-v0_96/package/scripts/params.py | 109 --
.../package/scripts/status_params.py | 26 -
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 62 --
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 62 --
.../package/templates/hbase-env.sh.j2 | 81 --
.../package/templates/hbase_client_jaas.conf.j2 | 22 -
.../package/templates/hbase_master_jaas.conf.j2 | 25 -
.../templates/hbase_regionserver_jaas.conf.j2 | 25 -
.../package/templates/regionservers.j2 | 20 -
app-packages/hbase-v0_96/resources.json | 19 -
app-packages/hbase/README.txt | 54 +
app-packages/hbase/appConfig.json | 64 ++
app-packages/hbase/configuration/global.xml | 160 +++
.../hbase/configuration/hbase-log4j.xml | 142 +++
.../hbase/configuration/hbase-policy.xml | 53 +
app-packages/hbase/configuration/hbase-site.xml | 365 +++++++
app-packages/hbase/jmx_metrics.json | 56 +
app-packages/hbase/metainfo.xml | 107 ++
app-packages/hbase/package/scripts/__init__.py | 19 +
app-packages/hbase/package/scripts/functions.py | 40 +
app-packages/hbase/package/scripts/hbase.py | 125 +++
.../hbase/package/scripts/hbase_client.py | 43 +
.../hbase/package/scripts/hbase_master.py | 63 ++
.../hbase/package/scripts/hbase_regionserver.py | 66 ++
.../hbase/package/scripts/hbase_service.py | 45 +
app-packages/hbase/package/scripts/params.py | 109 ++
.../hbase/package/scripts/status_params.py | 26 +
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 62 ++
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 62 ++
.../hbase/package/templates/hbase-env.sh.j2 | 81 ++
.../package/templates/hbase_client_jaas.conf.j2 | 22 +
.../package/templates/hbase_master_jaas.conf.j2 | 25 +
.../templates/hbase_regionserver_jaas.conf.j2 | 25 +
.../hbase/package/templates/regionservers.j2 | 20 +
app-packages/hbase/pom.xml | 93 ++
app-packages/hbase/resources.json | 19 +
app-packages/hbase/src/assembly/hbase.xml | 71 ++
app-packages/storm-v0_91/metainfo.xml | 229 ++--
pom.xml | 141 +--
slider-agent/conf/agent.ini | 2 +
.../src/main/python/agent/AgentConfig.py | 21 +
slider-agent/src/main/python/agent/Constants.py | 3 +
.../src/main/python/agent/Controller.py | 23 +-
slider-agent/src/main/python/agent/main.py | 4 +
.../src/test/python/agent/TestController.py | 30 +
slider-agent/src/test/python/agent/TestMain.py | 5 +-
slider-assembly/pom.xml | 11 +-
slider-assembly/src/main/scripts/slider | 1 -
slider-assembly/src/main/scripts/slider.py | 190 ++++
.../providers/AbstractProviderService.java | 15 +-
.../slider/providers/ProviderService.java | 5 +-
.../providers/agent/AgentClientProvider.java | 29 +-
.../slider/providers/agent/AgentKeys.java | 4 +-
.../providers/agent/AgentLaunchParameter.java | 130 +++
.../providers/agent/AgentProviderService.java | 232 +++-
.../slider/providers/agent/AgentRoles.java | 18 +-
.../apache/slider/providers/agent/Command.java | 13 +-
.../slider/providers/agent/CommandResult.java | 16 +-
.../providers/agent/ComponentInstanceState.java | 24 +
.../slider/providers/agent/ContainerState.java | 41 +
.../providers/agent/HeartbeatMonitor.java | 116 ++
.../agent/application/metadata/Application.java | 121 +++
.../agent/application/metadata/Metainfo.java | 17 +-
.../application/metadata/MetainfoParser.java | 10 +-
.../agent/application/metadata/Service.java | 121 ---
.../server/appmaster/AMViewForProviders.java | 27 +
.../server/appmaster/SliderAppMaster.java | 32 +-
.../slider/server/appmaster/state/AppState.java | 20 +
.../appmaster/state/ContainerPriority.java | 8 +-
.../appmaster/state/OutstandingRequest.java | 7 +-
.../server/appmaster/state/RoleHistory.java | 8 +
.../server/appmaster/web/SliderAmIpFilter.java | 2 +-
.../services/workflow/ForkedProcessService.java | 25 +-
.../services/workflow/LongLivedProcess.java | 13 +
.../test_command_log/appConfig_fast_no_reg.json | 29 +
.../test_command_log/appConfig_no_hb.json | 29 +
.../app_packages/test_command_log/metainfo.xml | 62 +-
.../agent/AgentMiniClusterTestBase.groovy | 41 +-
.../slider/providers/agent/AgentTestBase.groovy | 44 +-
.../agent/TestAgentAMManagementWS.groovy | 3 +-
.../slider/providers/agent/TestAgentEcho.groovy | 6 +-
.../providers/agent/TestBuildBasicAgent.groovy | 78 +-
.../curator/TestRegistryRestResources.groovy | 3 +-
.../model/appstate/TestMockRMOperations.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../publisher/TestPublisherRestResources.groovy | 3 +-
.../slider/common/tools/TestSliderUtils.java | 23 +-
.../agent/TestAgentClientProvider.java | 40 +-
.../agent/TestAgentLaunchParameter.java | 76 ++
.../agent/TestAgentProviderService.java | 61 +-
.../providers/agent/TestHeartbeatMonitor.java | 136 +++
.../metadata/MetainfoParserTest.java | 15 +-
.../publisher/TestAgentProviderService.java | 5 +-
.../TestWorkflowForkedProcessService.java | 3 +-
.../org/apache/slider/tools/TestUtility.java | 71 ++
slider-core/src/test/python/appdef_1.zip | Bin 972 -> 0 bytes
slider-core/src/test/python/metainfo.xml | 68 +-
.../src/test/resources/example-slider-test.xml | 4 +-
.../org/apache/slider/common/tools/test.zip | Bin 1273 -> 0 bytes
.../slider/common/tools/test/metainfo.txt | 16 +
.../slider/common/tools/test/metainfo.xml | 95 ++
.../slider/common/tools/test/someOtherFile.txt | 16 +
.../slider/common/tools/test/someOtherFile.xml | 16 +
.../agent/application/metadata/metainfo.xml | 136 ++-
slider-funtest/pom.xml | 38 +-
.../funtest/framework/AgentUploads.groovy | 68 ++
.../funtest/framework/CommandTestBase.groovy | 9 +-
.../funtest/framework/FileUploader.groovy | 100 ++
.../funtest/framework/FuntestProperties.groovy | 12 +-
.../slider/funtest/framework/SudoClosure.groovy | 54 +
.../lifecycle/AgentCommandTestBase.groovy | 135 ++-
.../lifecycle/TestAgentClusterLifecycle.groovy | 2 +-
.../funtest/lifecycle/TestAgentFailures.groovy | 103 ++
.../funtest/lifecycle/TestAgentFailures2.groovy | 103 ++
.../lifecycle/TestAppsThroughAgent.groovy | 111 +-
.../lifecycle/TestClusterBuildDestroy.groovy | 2 +-
.../accumulo/accumulo-funtests/pom.xml | 2 +-
.../slider/providers/accumulo/TestStub.groovy | 32 +
slider-providers/hbase/hbase-funtests/pom.xml | 2 +-
.../slider/providers/hbase/TestStub.groovy | 32 +
.../providers/hbase/HBaseProviderService.java | 13 +-
.../slider/providers/hbase/HBaseRoles.java | 3 +-
src/site/markdown/architecture/architecture.md | 142 ---
src/site/markdown/architecture/index.md | 27 -
src/site/markdown/architecture/rolehistory.md | 1010 ------------------
src/site/markdown/client-configuration.md | 310 ------
src/site/markdown/configuration/core.md | 407 -------
.../example-app_configuration-resolved.json | 42 -
.../example-app_configuration.json | 25 -
.../markdown/configuration/example-empty.json | 8 -
.../configuration/example-internal.json | 21 -
.../example-overridden-resolved.json | 25 -
.../configuration/example-overridden.json | 23 -
.../configuration/example-resources.json | 25 -
.../markdown/configuration/index-markdown.md | 31 -
src/site/markdown/configuration/index.md | 38 -
.../markdown/configuration/original-hbase.json | 139 ---
.../markdown/configuration/proposed-hbase.json | 273 -----
src/site/markdown/configuration/redesign.md | 478 ---------
.../configuration/resolved-resources.json | 22 -
.../markdown/configuration/specification.md | 512 ---------
src/site/markdown/debugging.md | 92 --
src/site/markdown/developing/building.md | 374 -------
.../markdown/developing/functional_tests.md | 416 --------
src/site/markdown/developing/index.md | 35 -
src/site/markdown/developing/manual_testing.md | 53 -
src/site/markdown/developing/releasing.md | 195 ----
src/site/markdown/developing/testing.md | 182 ----
src/site/markdown/examples.md | 159 ---
src/site/markdown/exitcodes.md | 161 ---
src/site/markdown/getting_started.md | 509 ---------
src/site/markdown/index.md | 94 --
src/site/markdown/manpage.md | 483 ---------
.../registry/a_YARN_service_registry.md | 226 ----
src/site/markdown/registry/index.md | 47 -
.../registry/initial_registry_design.md | 110 --
.../markdown/registry/p2p_service_registries.md | 137 ---
src/site/markdown/registry/references.md | 49 -
src/site/markdown/registry/registry-model.md | 75 --
.../service_registry_end_to_end_scenario.md | 156 ---
...lication_registration_and_binding_problem.md | 192 ----
.../markdown/release_notes/release-0.30.0.md | 36 -
src/site/markdown/security.md | 197 ----
.../slider_specs/application_configuration.md | 82 --
.../slider_specs/application_definition.md | 182 ----
.../application_instance_configuration.md | 104 --
.../markdown/slider_specs/application_needs.md | 140 ---
.../slider_specs/application_package.md | 147 ---
.../slider_specs/canonical_scenarios.md | 165 ---
.../slider_specs/creating_app_definitions.md | 123 ---
src/site/markdown/slider_specs/index.md | 53 -
.../slider_specs/resource_specification.md | 53 -
.../slider_specs/writing_app_command_scripts.md | 211 ----
src/site/markdown/specification/cli-actions.md | 675 ------------
src/site/markdown/specification/index.md | 41 -
src/site/markdown/specification/slider-model.md | 286 -----
src/site/markdown/troubleshooting.md | 154 ---
src/site/resources/hoya_am_architecture.png | Bin 137154 -> 0 bytes
.../resources/images/app_config_folders_01.png | Bin 21050 -> 0 bytes
.../resources/images/app_package_sample_04.png | Bin 67927 -> 0 bytes
src/site/resources/images/image_0.png | Bin 194914 -> 0 bytes
src/site/resources/images/image_1.png | Bin 77882 -> 0 bytes
src/site/resources/images/managed_client.png | Bin 71352 -> 0 bytes
src/site/resources/images/slider-container.png | Bin 70265 -> 0 bytes
src/site/resources/images/unmanaged_client.png | Bin 58813 -> 0 bytes
src/site/site.xml | 84 --
src/test/clusters/c6401/slider/log4j.properties | 83 ++
.../clusters/c6401/slider/slider-client.xml | 71 ++
275 files changed, 7261 insertions(+), 14799 deletions(-)
----------------------------------------------------------------------
[38/50] [abbrv] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by st...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9619c335
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9619c335
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9619c335
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 9619c3356f44e91c4cb6de604e0ecfd8c1abcac9
Parents: 7089fa0 93bd8ce
Author: Steve Loughran <st...@apache.org>
Authored: Wed Jun 25 11:23:08 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Jun 25 11:23:08 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/web/SliderAmIpFilter.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[42/50] [abbrv] git commit: SLIDER-96. Fixes to some tests.
Posted by st...@apache.org.
SLIDER-96. Fixes to some tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/78a1364b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/78a1364b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/78a1364b
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 78a1364b1c52453f029d3afdedfd1fc6e9fb6ebb
Parents: b64e1cd
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Jun 26 07:21:28 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jun 26 13:41:31 2014 -0700
----------------------------------------------------------------------
.../slider/providers/agent/AgentTestBase.groovy | 2 +-
.../providers/agent/TestBuildBasicAgent.groovy | 73 ++++++++++----------
2 files changed, 36 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/78a1364b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index bb3eabe..9b4c377 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -47,7 +47,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
- public static String app_def_pkg_path;
+ public String app_def_pkg_path;
@Before
public void setupAppPkg() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/78a1364b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 1879f84..9f44888 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -38,34 +38,31 @@ import static org.apache.slider.providers.agent.AgentKeys.*
@Slf4j
class TestBuildBasicAgent extends AgentTestBase {
static String TEST_FILES = "./src/test/resources/org/apache/slider/providers/agent/tests/"
+ static File slider_core = new File(new File(".").absoluteFile, "src/test/python");
+ static String bad_app_def = "appdef_1.tar"
+ static File bad_app_def_path = new File(slider_core, bad_app_def)
+ static String agt_conf = "agent.ini"
+ static File agt_conf_path = new File(slider_core, agt_conf)
@Override
void checkTestAssumptions(YarnConfiguration conf) {
}
- private static class TestResources {
- static File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- static String bad_app_def = "appdef_1.tar"
- static File bad_app_def_path = new File(slider_core, bad_app_def)
- static String agt_conf = "agent.ini"
- static File agt_conf_path = new File(slider_core, agt_conf)
-
- public static File getAppDef() {
- return new File(app_def_pkg_path);
- }
+ private File getAppDef() {
+ return new File(app_def_pkg_path);
+ }
- public static File getBadAppDef() {
- return bad_app_def_path;
- }
+ private File getBadAppDef() {
+ return bad_app_def_path;
+ }
- public static File getAgentConf() {
- return agt_conf_path;
- }
+ private File getAgentConf() {
+ return agt_conf_path;
+ }
- public static File getAgentImg() {
- return new File(app_def_pkg_path);
- }
+ private File getAgentImg() {
+ return new File(app_def_pkg_path);
}
@@ -86,8 +83,8 @@ class TestBuildBasicAgent extends AgentTestBase {
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_PACKAGE, ".",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_OPTION, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1",
@@ -106,8 +103,8 @@ class TestBuildBasicAgent extends AgentTestBase {
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_OPTION, PACKAGE_PATH, ".",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_COMP_OPT, master, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_COMP_OPT, rs, SCRIPT_PATH, "agent/scripts/agent.py",
ARG_RES_COMP_OPT, master, ResourceKeys.COMPONENT_PRIORITY, "2",
@@ -188,8 +185,8 @@ class TestBuildBasicAgent extends AgentTestBase {
(rs): 5
],
[
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_PACKAGE, ".",
ARG_COMP_OPT, SliderKeys.COMPONENT_AM, RoleKeys.JVM_OPTS, jvmopts,
ARG_COMP_OPT, master, RoleKeys.JVM_OPTS, jvmopts,
@@ -218,8 +215,8 @@ class TestBuildBasicAgent extends AgentTestBase {
"role": 1,
],
[
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_PACKAGE, ".",
ARG_RES_COMP_OPT, "role", ResourceKeys.COMPONENT_PRIORITY, "3",
],
@@ -255,8 +252,8 @@ class TestBuildBasicAgent extends AgentTestBase {
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -273,9 +270,9 @@ class TestBuildBasicAgent extends AgentTestBase {
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_IMAGE, "file://" + TestResources.getAgentImg().absolutePath + ".badfile",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_IMAGE, "file://" + getAgentImg().absolutePath + ".badfile",
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -291,7 +288,7 @@ class TestBuildBasicAgent extends AgentTestBase {
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -308,7 +305,7 @@ class TestBuildBasicAgent extends AgentTestBase {
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_PACKAGE, ".",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -324,9 +321,9 @@ class TestBuildBasicAgent extends AgentTestBase {
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_PACKAGE, ".",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getBadAppDef().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getBadAppDef().absolutePath,
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
],
@@ -356,8 +353,8 @@ class TestBuildBasicAgent extends AgentTestBase {
[:],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
- ARG_OPTION, APP_DEF, "file://" + TestResources.getAppDef().absolutePath,
- ARG_OPTION, AGENT_CONF, "file://" + TestResources.getAgentConf().absolutePath,
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_PACKAGE, ".",
ARG_RESOURCES, TEST_FILES + "good/resources.json",
ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
[43/50] [abbrv] git commit: SLIDER-96. Related fix. Dynamically
create app packages for tests.
Posted by st...@apache.org.
SLIDER-96. Related fix. Dynamically create app packages for tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b64e1cdf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b64e1cdf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b64e1cdf
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: b64e1cdfc5d4d5c89c786a301209bd43391d5b36
Parents: 4eb8ea9
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jun 25 14:07:59 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Jun 26 13:41:31 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/metainfo.xml | 238 ++++++++++---------
app-packages/hbase/metainfo.xml | 168 +++++++------
app-packages/storm-v0_91/metainfo.xml | 229 +++++++++---------
.../providers/agent/AgentClientProvider.java | 23 +-
.../agent/AgentMiniClusterTestBase.groovy | 41 +++-
.../slider/providers/agent/AgentTestBase.groovy | 44 +++-
.../agent/TestAgentAMManagementWS.groovy | 3 +-
.../slider/providers/agent/TestAgentEcho.groovy | 6 +-
.../providers/agent/TestBuildBasicAgent.groovy | 17 +-
.../curator/TestRegistryRestResources.groovy | 3 +-
.../publisher/TestPublisherRestResources.groovy | 3 +-
slider-core/src/test/python/appdef_1.zip | Bin 972 -> 0 bytes
slider-core/src/test/python/metainfo.xml | 68 +++---
13 files changed, 445 insertions(+), 398 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/app-packages/accumulo/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml
index eba75c6..4cf6c79 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -17,129 +17,131 @@
-->
<metainfo>
<schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ACCUMULO</name>
- <comment>
- The Apache Accumulo sorted, distributed key/value store is a robust,
- scalable, high performance data storage system that features cell-based
- access control and customizable server-side processing. It is based on
- Google's BigTable design and is built on top of Apache Hadoop,
- Zookeeper, and Thrift.
- Requirements:
- 1. Ensure parent dir for path (accumulo-site/instance.volumes) is accessible to the App owner.
- </comment>
- <version>${accumulo.version}</version>
- <exportGroups>
- <exportGroup>
- <name>QuickLinks</name>
- <exports>
- <export>
- <name>org.apache.slider.monitor</name>
- <value>${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}</value>
- </export>
- <export>
- <name>org.apache.slider.jmx</name>
- <value>${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml</value>
- </export>
- </exports>
- </exportGroup>
- </exportGroups>
- <commandOrders>
- <commandOrder>
- <command>ACCUMULO_MASTER-START</command>
- <requires>ACCUMULO_MONITOR-INSTALLED</requires>
- </commandOrder>
- <commandOrder>
- <command>ACCUMULO_TSERVER-START</command>
- <requires>ACCUMULO_MASTER-STARTED</requires>
- </commandOrder>
- <commandOrder>
- <command>ACCUMULO_MONITOR-START</command>
- <requires>ACCUMULO_MASTER-STARTED</requires>
- </commandOrder>
- <commandOrder>
- <command>ACCUMULO_GC-START</command>
- <requires>ACCUMULO_MASTER-STARTED</requires>
- </commandOrder>
- <commandOrder>
- <command>ACCUMULO_TRACER-START</command>
- <requires>ACCUMULO_MASTER-STARTED</requires>
- </commandOrder>
- </commandOrders>
- <components>
- <component>
- <name>ACCUMULO_MASTER</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/accumulo_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <application>
+ <name>ACCUMULO</name>
+ <comment>
+ The Apache Accumulo sorted, distributed key/value store is a robust,
+ scalable, high performance data storage system that features cell-based
+ access control and customizable server-side processing. It is based on
+ Google's BigTable design and is built on top of Apache Hadoop,
+ Zookeeper, and Thrift.
+ Requirements:
+ 1. Ensure parent dir for path (accumulo-site/instance.volumes) is accessible to the App owner.
+ </comment>
+ <version>${accumulo.version}</version>
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>org.apache.slider.monitor</name>
+ <value>
+ ${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}
+ </value>
+ </export>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>
+ ${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml
+ </value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+ <commandOrders>
+ <commandOrder>
+ <command>ACCUMULO_MASTER-START</command>
+ <requires>ACCUMULO_MONITOR-INSTALLED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>ACCUMULO_TSERVER-START</command>
+ <requires>ACCUMULO_MASTER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>ACCUMULO_MONITOR-START</command>
+ <requires>ACCUMULO_MASTER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>ACCUMULO_GC-START</command>
+ <requires>ACCUMULO_MASTER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>ACCUMULO_TRACER-START</command>
+ <requires>ACCUMULO_MASTER-STARTED</requires>
+ </commandOrder>
+ </commandOrders>
+ <components>
+ <component>
+ <name>ACCUMULO_MASTER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>ACCUMULO_MONITOR</name>
- <category>MASTER</category>
- <publishConfig>true</publishConfig>
- <commandScript>
- <script>scripts/accumulo_monitor.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>ACCUMULO_MONITOR</name>
+ <category>MASTER</category>
+ <publishConfig>true</publishConfig>
+ <commandScript>
+ <script>scripts/accumulo_monitor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>ACCUMULO_GC</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/accumulo_gc.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>ACCUMULO_GC</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_gc.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>ACCUMULO_TRACER</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/accumulo_tracer.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>ACCUMULO_TRACER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/accumulo_tracer.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>ACCUMULO_TSERVER</name>
- <category>SLAVE</category>
- <commandScript>
- <script>scripts/accumulo_tserver.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
+ <component>
+ <name>ACCUMULO_TSERVER</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/accumulo_tserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
- <component>
- <name>ACCUMULO_CLIENT</name>
- <category>CLIENT</category>
- <commandScript>
- <script>scripts/accumulo_client.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
- </components>
+ <component>
+ <name>ACCUMULO_CLIENT</name>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/accumulo_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+ </components>
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/accumulo-${accumulo.version}-bin.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/accumulo-${accumulo.version}-bin.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
- </service>
- </services>
+ </application>
</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index d5bb2d5..99413f6 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -17,93 +17,91 @@
-->
<metainfo>
<schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>HBASE</name>
- <comment>
- Apache HBase is the Hadoop database, a distributed, scalable, big data store.
- Requirements:
- 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner.
- 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance.
- </comment>
- <version>${hbase.version}</version>
- <type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
- <exportGroups>
- <exportGroup>
- <name>QuickLinks</name>
- <exports>
- <export>
- <name>org.apache.slider.jmx</name>
- <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
- </export>
- <export>
- <name>org.apache.slider.monitor</name>
- <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
- </export>
- <export>
- <name>org.apache.slider.metrics</name>
- <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
- </export>
- <export>
- <name>org.apache.slider.ganglia</name>
- <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
- </export>
- </exports>
- </exportGroup>
- </exportGroups>
- <commandOrders>
- <commandOrder>
- <command>HBASE_REGIONSERVER-START</command>
- <requires>HBASE_MASTER-STARTED</requires>
- </commandOrder>
- </commandOrders>
- <components>
- <component>
- <name>HBASE_MASTER</name>
- <category>MASTER</category>
- <minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
- <commandScript>
- <script>scripts/hbase_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <application>
+ <name>HBASE</name>
+ <comment>
+ Apache HBase is the Hadoop database, a distributed, scalable, big data store.
+ Requirements:
+ 1. Ensure parent dir for path (hbase-site/hbase.rootdir) is accessible to the App owner.
+ 2. Ensure ZK root (hbase-site/zookeeper.znode.parent) is unique for the App instance.
+ </comment>
+ <version>${hbase.version}</version>
+ <type>YARN-APP</type>
+ <minHadoopVersion>2.1.0</minHadoopVersion>
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
+ </export>
+ <export>
+ <name>org.apache.slider.monitor</name>
+ <value>http://${HBASE_MASTER_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
+ </export>
+ <export>
+ <name>org.apache.slider.metrics</name>
+ <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.ganglia</name>
+ <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+ <commandOrders>
+ <commandOrder>
+ <command>HBASE_REGIONSERVER-START</command>
+ <requires>HBASE_MASTER-STARTED</requires>
+ </commandOrder>
+ </commandOrders>
+ <components>
+ <component>
+ <name>HBASE_MASTER</name>
+ <category>MASTER</category>
+ <minInstanceCount>1</minInstanceCount>
+ <maxInstanceCount>2</maxInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_master.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>HBASE_REGIONSERVER</name>
- <category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
- <commandScript>
- <script>scripts/hbase_regionserver.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
+ <component>
+ <name>HBASE_REGIONSERVER</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_regionserver.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
- <component>
- <name>HBASE_CLIENT</name>
- <category>CLIENT</category>
- <minInstanceCount>0</minInstanceCount>
- <commandScript>
- <script>scripts/hbase_client.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- </component>
- </components>
+ <component>
+ <name>HBASE_CLIENT</name>
+ <category>CLIENT</category>
+ <minInstanceCount>0</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+ </components>
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/hbase-${hbase.version}-bin.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/hbase-${hbase.version}-bin.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
- </service>
- </services>
+ </application>
</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/app-packages/storm-v0_91/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/metainfo.xml b/app-packages/storm-v0_91/metainfo.xml
index 2b10c52..7edd794 100644
--- a/app-packages/storm-v0_91/metainfo.xml
+++ b/app-packages/storm-v0_91/metainfo.xml
@@ -18,129 +18,128 @@
<metainfo>
<schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>STORM</name>
- <comment>Apache Hadoop Stream processing framework</comment>
- <version>0.9.1.2.1</version>
+ <application>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
- <exportGroups>
- <exportGroup>
- <name>QuickLinks</name>
- <exports>
- <export>
- <name>org.apache.slider.jmx</name>
- <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
- </export>
- <export>
- <name>org.apache.slider.monitor</name>
- <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
- </export>
- <export>
- <name>org.apache.slider.metrics</name>
- <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
- </export>
- <export>
- <name>org.apache.slider.ganglia</name>
- <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
- </export>
- </exports>
- </exportGroup>
- </exportGroups>
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
+ </export>
+ <export>
+ <name>org.apache.slider.monitor</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.metrics</name>
+ <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.ganglia</name>
+ <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
- <commandOrders>
- <commandOrder>
- <command>NIMBUS-START</command>
- <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED</requires>
- </commandOrder>
- <commandOrder>
- <command>SUPERVISOR-START</command>
- <requires>NIMBUS-STARTED</requires>
- </commandOrder>
- <commandOrder>
- <command>DRPC_SERVER-START</command>
- <requires>NIMBUS-STARTED</requires>
- </commandOrder>
- <commandOrder>
- <command>STORM_REST_API-START</command>
- <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
- </commandOrder>
- <commandOrder>
- <command>STORM_UI_SERVER-START</command>
- <requires>NIMBUS-STARTED</requires>
- </commandOrder>
- </commandOrders>
+ <commandOrders>
+ <commandOrder>
+ <command>NIMBUS-START</command>
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+ </requires>
+ </commandOrder>
+ <commandOrder>
+ <command>SUPERVISOR-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>DRPC_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_REST_API-START</command>
+ <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_UI_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ </commandOrders>
- <components>
+ <components>
- <component>
- <name>NIMBUS</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/nimbus.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>STORM_REST_API</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/rest_api.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>SUPERVISOR</name>
- <category>SLAVE</category>
- <commandScript>
- <script>scripts/supervisor.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>STORM_UI_SERVER</name>
- <category>MASTER</category>
- <publishConfig>true</publishConfig>
- <commandScript>
- <script>scripts/ui_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <publishConfig>true</publishConfig>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
- <component>
- <name>DRPC_SERVER</name>
- <category>MASTER</category>
- <commandScript>
- <script>scripts/drpc_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
- <configuration-dependencies>
- <config-type>storm-site</config-type>
- <config-type>global</config-type>
- </configuration-dependencies>
- </service>
- </services>
+ <configuration-dependencies>
+ <config-type>storm-site</config-type>
+ <config-type>global</config-type>
+ </configuration-dependencies>
+ </application>
</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index ea1c1b7..f3e2dd7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -202,18 +202,25 @@ public class AgentClientProvider extends AbstractClientProvider
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
String appDef) throws SliderException {
Set<String> tags;
+ Metainfo metainfo;
try {
- Metainfo metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef);
- Application application = metainfo.getApplication();
- tags = new HashSet<>();
- tags.add("Name: " + application.getName());
- tags.add("Version: " + application.getVersion());
- tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
+ metainfo = AgentUtils.getApplicationMetainfo(fileSystem, appDef);
} catch (IOException e) {
- log.error("error retrieving metainfo from {}", appDef, e);
- throw new SliderException("error retrieving metainfo", e);
+ log.error("Error retrieving metainfo from {}", appDef, e);
+ throw new SliderException("Error retrieving metainfo", e);
}
+ if(metainfo == null) {
+ log.error("Error retrieving metainfo from {}", appDef);
+ throw new SliderException("Error parsing metainfo file, possibly bad structure.");
+ }
+
+ Application application = metainfo.getApplication();
+ tags = new HashSet<>();
+ tags.add("Name: " + application.getName());
+ tags.add("Version: " + application.getVersion());
+ tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
+
return tags;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index cea3d74..8a4e5d8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -20,13 +20,19 @@ package org.apache.slider.agent
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.utils.IOUtils
+import org.apache.commons.io.FileUtils
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.AfterClass
import org.junit.BeforeClass
+import org.junit.rules.TemporaryFolder
/**
* test base for agent clusters
@@ -34,28 +40,52 @@ import org.junit.BeforeClass
@CompileStatic
@Slf4j
public abstract class AgentMiniClusterTestBase
- extends YarnZKMiniClusterTestBase {
+extends YarnZKMiniClusterTestBase {
protected static File agentConf
protected static File agentDef
protected static File imagePath
- protected static Map<String, String> agentDefOptions
+ protected static Map<String, String> agentDefOptions
+ private static TemporaryFolder tempFolder = new TemporaryFolder();
@BeforeClass
public static void createSubConfFiles() {
+
File destDir = new File("target/agent_minicluster_testbase")
destDir.mkdirs()
agentConf = new File(destDir, "agentconf.zip")
agentConf.createNewFile()
agentDef = new File(destDir, "agentdef")
agentDef.createNewFile()
- File slider_dir = new File(new File(".").absoluteFile, "src/test/python");
- imagePath = new File(slider_dir, "appdef_1.zip")
+
+ // dynamically create the app package for the test
+ tempFolder.create()
+ def pkgPath = tempFolder.newFolder("testpkg")
+ File imagePath = new File(pkgPath, "appdef_1.zip").canonicalFile
+ File metainfo = new File(new File(".").absoluteFile, "src/test/python/metainfo.xml");
+ ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new FileOutputStream(imagePath));
+ try {
+ zipFile.putArchiveEntry(new ZipArchiveEntry(metainfo.name));
+ IOUtils.copy(new FileInputStream(metainfo), zipFile);
+ zipFile.closeArchiveEntry();
+ }
+ finally {
+ zipFile.close();
+ }
+
agentDefOptions = [
- (AgentKeys.APP_DEF) : imagePath.toURI().toString(),
+ (AgentKeys.APP_DEF): imagePath.toURI().toString(),
(AgentKeys.AGENT_CONF): agentConf.toURI().toString()
]
}
+ @AfterClass
+ public static void cleanSubConfFiles() {
+ if (tempFolder.getRoot().exists()) {
+ FileUtils.deleteDirectory(tempFolder.getRoot());
+ }
+ }
+
+
@Override
public String getTestConfigurationPath() {
return "src/main/resources/" + AgentKeys.CONF_RESOURCE;
@@ -74,7 +104,6 @@ public abstract class AgentMiniClusterTestBase
void teardown() {
super.teardown();
if (teardownKillall) {
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index b6d0c6c..bb3eabe 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -20,11 +20,17 @@ package org.apache.slider.providers.agent
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream
+import org.apache.commons.compress.utils.IOUtils
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.Before
+import org.junit.Rule
+import org.junit.rules.TemporaryFolder
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
@@ -36,17 +42,31 @@ import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
@Slf4j
public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
- public static
- final int AGENT_CLUSTER_STARTUP_TIME = 1000 * DEFAULT_AGENT_LAUNCH_TIME_SECONDS
-
- /**
- * The time to sleep before trying to talk to the HBase Master and
- * expect meaningful results.
- */
- public static
- final int AGENT_CLUSTER_STARTUP_TO_LIVE_TIME = AGENT_CLUSTER_STARTUP_TIME
- public static final int AGENT_GO_LIVE_TIME = 60000
-
+ public static final int AGENT_CLUSTER_STARTUP_TIME = 1000 * DEFAULT_AGENT_LAUNCH_TIME_SECONDS
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ public static String app_def_pkg_path;
+
+ @Before
+ public void setupAppPkg() {
+ if (app_def_pkg_path == null) {
+ def pkgPath = folder.newFolder("testpkg")
+ File zipFileName = new File(pkgPath, "appdef_1.zip").canonicalFile
+ File metainfo = new File(new File(".").absoluteFile, "src/test/python/metainfo.xml");
+ ZipArchiveOutputStream zipFile = new ZipArchiveOutputStream(new FileOutputStream(zipFileName));
+ try {
+ zipFile.putArchiveEntry(new ZipArchiveEntry(metainfo.name));
+ IOUtils.copy(new FileInputStream(metainfo), zipFile);
+ zipFile.closeArchiveEntry();
+ }
+ finally {
+ zipFile.close();
+ }
+ app_def_pkg_path = zipFileName.absolutePath
+ }
+ }
@Override
public String getTestConfigurationPath() {
@@ -102,7 +122,7 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase {
boolean deleteExistingData,
boolean create,
boolean blockUntilRunning) {
-
+
YarnConfiguration conf = testConfiguration
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index de20e10..51d9bd6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -58,8 +58,7 @@ class TestAgentAMManagementWS extends AgentTestBase {
false)
Map<String, Integer> roles = [:]
File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- String app_def = "appdef_1.zip"
- File app_def_path = new File(slider_core, app_def)
+ File app_def_path = new File(app_def_pkg_path)
String agt_ver = "version"
File agt_ver_path = new File(slider_core, agt_ver)
String agt_conf = "agent.ini"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 1072ebe..2f03b09 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -36,10 +36,9 @@ import static org.apache.slider.providers.agent.AgentKeys.*
@Slf4j
class TestAgentEcho extends AgentTestBase {
-
@Override
void checkTestAssumptions(YarnConfiguration conf) {
-
+
}
@Test
@@ -57,8 +56,7 @@ class TestAgentEcho extends AgentTestBase {
File slider_core = new File(new File(".").absoluteFile, "src/test/python");
String echo_py = "echo.py"
File echo_py_path = new File(slider_core, echo_py)
- String app_def = "appdef_1.zip"
- File app_def_path = new File(slider_core, app_def)
+ File app_def_path = new File(app_def_pkg_path)
String agt_ver = "version"
File agt_ver_path = new File(slider_core, agt_ver)
String agt_conf = "agent.ini"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index a597707..1879f84 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -46,30 +46,29 @@ class TestBuildBasicAgent extends AgentTestBase {
private static class TestResources {
static File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- static String app_def = "appdef_1.zip"
static String bad_app_def = "appdef_1.tar"
- static File app_def_path = new File(slider_core, app_def)
static File bad_app_def_path = new File(slider_core, bad_app_def)
static String agt_conf = "agent.ini"
static File agt_conf_path = new File(slider_core, agt_conf)
- static public File getAppDef() {
- return app_def_path;
+ public static File getAppDef() {
+ return new File(app_def_pkg_path);
}
- static public File getBadAppDef() {
+ public static File getBadAppDef() {
return bad_app_def_path;
}
- static public File getAgentConf() {
+ public static File getAgentConf() {
return agt_conf_path;
}
- static public File getAgentImg() {
- return app_def_path;
+ public static File getAgentImg() {
+ return new File(app_def_pkg_path);
}
}
+
@Test
public void testBuildMultipleRoles() throws Throwable {
@@ -265,7 +264,7 @@ class TestBuildBasicAgent extends AgentTestBase {
false)
failWithBuildSucceeding(badArgs1, "missing package home or image path")
} catch (BadConfigException expected) {
-
+
}
try {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
index 303cc73..4cc0f08 100644
--- a/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/curator/TestRegistryRestResources.groovy
@@ -75,8 +75,7 @@ class TestRegistryRestResources extends AgentTestBase {
false)
Map<String, Integer> roles = [:]
File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- String app_def = "appdef_1.zip"
- File app_def_path = new File(slider_core, app_def)
+ File app_def_path = new File(app_def_pkg_path)
String agt_ver = "version"
File agt_ver_path = new File(slider_core, agt_ver)
String agt_conf = "agent.ini"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 1309e57..cd7c9d8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -58,8 +58,7 @@ class TestPublisherRestResources extends AgentTestBase {
false)
Map<String, Integer> roles = [:]
File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- String app_def = "appdef_1.zip"
- File app_def_path = new File(slider_core, app_def)
+ File app_def_path = new File(app_def_pkg_path)
String agt_ver = "version"
File agt_ver_path = new File(slider_core, agt_ver)
String agt_conf = "agent.ini"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/python/appdef_1.zip
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/appdef_1.zip b/slider-core/src/test/python/appdef_1.zip
deleted file mode 100644
index 6ee6af4..0000000
Binary files a/slider-core/src/test/python/appdef_1.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b64e1cdf/slider-core/src/test/python/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index beefd9c..09b314e 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -17,41 +17,39 @@
-->
<metainfo>
<schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>ECHO</name>
- <comment>
- Echo
- </comment>
- <version>0.1</version>
- <type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
- <components>
- <component>
- <name>echo</name>
- <category>MASTER</category>
- <minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
- <commandScript>
- <script>echo.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osType>any</osType>
- <packages>
- <package>
- <type>tarball</type>
- <name>files/echo.tar.gz</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
+ <application>
+ <name>ECHO</name>
+ <comment>
+ Echo
+ </comment>
+ <version>0.1</version>
+ <type>YARN-APP</type>
+ <minHadoopVersion>2.1.0</minHadoopVersion>
+ <components>
+ <component>
+ <name>echo</name>
+ <category>MASTER</category>
+ <minInstanceCount>1</minInstanceCount>
+ <maxInstanceCount>2</maxInstanceCount>
+ <commandScript>
+ <script>echo.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/echo.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
- </service>
- </services>
+ </application>
</metainfo>
[45/50] [abbrv] git commit: SLIDER-173 relaxLocality should be false
if specific node is requested
Posted by st...@apache.org.
SLIDER-173 relaxLocality should be false if specific node is requested
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/590cbe02
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/590cbe02
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/590cbe02
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 590cbe02a805362242657f1c3be3b60fe0ab7825
Parents: ea251f1
Author: tedyu <yu...@gmail.com>
Authored: Thu Jun 26 19:23:35 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jun 26 19:23:35 2014 -0700
----------------------------------------------------------------------
.../slider/server/appmaster/state/ContainerPriority.java | 8 ++++++--
.../slider/server/appmaster/state/OutstandingRequest.java | 7 ++-----
.../appmaster/model/appstate/TestMockRMOperations.groovy | 2 +-
3 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/590cbe02/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
index ccd9a64..935c09f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
@@ -37,9 +37,13 @@ import org.apache.hadoop.yarn.util.Records;
*/
public final class ContainerPriority {
+ // bit that represents whether location is specified
+ static final int LOCATION = 1 << 30;
+
public static int buildPriority(int role,
boolean locationSpecified) {
- return (role) ;
+ int location = locationSpecified ? LOCATION : 0;
+ return role | LOCATION;
}
@@ -53,7 +57,7 @@ public final class ContainerPriority {
public static int extractRole(int priority) {
- return priority ;
+ return priority >= LOCATION ? priority^LOCATION : priority;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/590cbe02/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 7d3e427..0d8b56c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -103,23 +103,20 @@ public final class OutstandingRequest {
RoleStatus role, long time) {
String[] hosts;
boolean relaxLocality;
- boolean locationSpecified;
requestedTime = time;
if (node != null) {
hosts = new String[1];
hosts[0] = node.hostname;
- relaxLocality = true;
- locationSpecified = true;
+ relaxLocality = false;
// tell the node it is in play
node.getOrCreate(roleId);
log.info("Submitting request for container on {}", hosts[0]);
} else {
hosts = null;
relaxLocality = true;
- locationSpecified = false;
}
Priority pri = ContainerPriority.createPriority(roleId,
- locationSpecified);
+ !relaxLocality);
AMRMClient.ContainerRequest request =
new AMRMClient.ContainerRequest(resource,
hosts,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/590cbe02/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
index 7f92f9c..168ac9f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
@@ -41,7 +41,7 @@ class TestMockRMOperations extends BaseMockAppStateTest implements MockRoles {
@Test
public void testPriorityOnly() throws Throwable {
- assert 5 == buildPriority(5, false)
+ assert 5 == extractRole(buildPriority(5, false))
}
@Test
[16/50] [abbrv] git commit: File upload work ... security model wrong
Posted by st...@apache.org.
File upload work ... security model wrong
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8327e52a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8327e52a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8327e52a
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 8327e52ae2dec00e0a954da1804413b1ef76f93c
Parents: 81578df
Author: Steve Loughran <st...@apache.org>
Authored: Mon Jun 23 18:42:25 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Jun 23 18:42:25 2014 -0700
----------------------------------------------------------------------
.../funtest/framework/AgentUploads.groovy | 41 +++++++++++
.../funtest/framework/FileUploader.groovy | 76 ++++++++++++++++++++
.../slider/funtest/framework/SudoClosure.groovy | 54 ++++++++++++++
3 files changed, 171 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8327e52a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
new file mode 100644
index 0000000..b1c29c0
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.UserGroupInformation
+
+@Slf4j
+class AgentUploads {
+ final Configuration conf
+
+
+ AgentUploads(Configuration conf) {
+ this.conf = conf
+ }
+
+ def execUploadSequence() {
+ UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn")
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8327e52a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
new file mode 100644
index 0000000..dd42473
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.permission.FsPermission
+import org.apache.hadoop.security.UserGroupInformation
+
+@Slf4j
+class FileUploader {
+ final Configuration conf
+ final UserGroupInformation user
+
+ FileUploader(Configuration conf, UserGroupInformation user) {
+ this.conf = conf
+ this.user = user
+ }
+
+ /**
+ * Copy if the file is considered out of date
+ * @param src
+ * @param dest
+ * @param force
+ * @return
+ */
+ public boolean copyIfOutOfDate(File src, Path dest, boolean force) {
+ def srcLen = src.length()
+ def fs = getFileSystem(user, dest.toUri())
+ boolean toCopy = force
+ if (!toCopy) {
+ try {
+ def status = fs.getFileStatus(dest)
+ toCopy = status.len != srcLen
+ } catch (FileNotFoundException fnfe) {
+ toCopy = true;
+ }
+ }
+ if (toCopy) {
+ log.info("Copying $src to $dest")
+ fs.mkdirs(dest, FsPermission.dirDefault)
+ return FileUtil.copy(src, fs, dest, false, conf)
+ } else {
+ log.debug("Skipping copy as the destination $dest considered up to date")
+ return false;
+ }
+
+ }
+
+
+ public static def getFileSystem(
+ UserGroupInformation user, final URI uri) {
+ SudoClosure.sudo(user) {
+ org.apache.hadoop.fs.FileSystem.get(uri, conf);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8327e52a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy
new file mode 100644
index 0000000..363e1b3
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SudoClosure.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.framework
+
+import org.apache.hadoop.security.UserGroupInformation
+
+import java.security.PrivilegedExceptionAction
+
+/**
+ * Bridge from groovy closures to doAs
+ * @param < T >
+ */
+class SudoClosure<T> implements PrivilegedExceptionAction<T> {
+
+ final Closure<T> closure;
+
+ SudoClosure(Closure<T> closure) {
+ this.closure = closure
+ }
+
+ @Override
+ T run() throws Exception {
+ return closure()
+ }
+
+ /**
+ *
+ * @param user
+ * @param closure
+ * @return
+ */
+ public static <T2> T2 sudo(UserGroupInformation user,
+ Closure<T2> closure) {
+
+ user.doAs(new SudoClosure<T2>(closure))
+
+ }
+}
[22/50] [abbrv] SLIDER-121 removed site documentation from git source
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/developing/building.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/developing/building.md b/src/site/markdown/developing/building.md
deleted file mode 100644
index ad36393..0000000
--- a/src/site/markdown/developing/building.md
+++ /dev/null
@@ -1,374 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Building Apache Slider
-
-
-Here's how to set this up.
-
-## Before you begin
-
-### Networking
-
-The network on the development system must be functional, with hostname lookup
-of the local host working. Tests will fail without this.
-
-### Java 7
-
-Slider is built on Java 7 -please have a JDK for Java 7 or 8 set up
-
-### Maven
-
-You will need a version of Maven 3.0+, set up with enough memory
-
- MAVEN_OPTS=-Xms256m -Xmx512m -Djava.awt.headless=true
-
-
-*Important*: As of October 6, 2013, Maven 3.1 is not supported due to
-[version issues](https://cwiki.apache.org/confluence/display/MAVEN/AetherClassNotFound).
-
-### Protoc
-
-You need a copy of the `protoc` compiler for protobuf compilation
-
-1. OS/X: `brew install protobuf`
-1. Others: consult (Building Hadoop documentation)[http://wiki.apache.org/hadoop/HowToContribute].
-
-The version of `protoc` installed must be the same as that used by Hadoop itself.
-This is absolutely critical to prevent JAR version problems.
-
-## Building a compatible Hadoop version
-
-
-Slider is built against Hadoop 2 -you can download and install
-a copy from the [Apache Hadoop Web Site](http://hadoop.apache.org).
-
-
-During development, its convenient (but not mandatory)
-to have a local version of Hadoop -so that we can find and fix bugs/add features in
-Hadoop as well in Slider.
-
-
-To build and install locally, check out apache svn/github, branch `release-2.4.0`,
-and create a branch off that tag
-
- git clone git://git.apache.org/hadoop-common.git
- cd hadoop-common
- git remote rename origin apache
- git fetch --tags apache
- git checkout release-2.4.0 --
- git checkout -b release-2.4.0
-
-
-For the scripts below, set the `HADOOP_VERSION` variable to the version
-
- export HADOOP_VERSION=2.4.0
-
-or, for building against a pre-release version of Hadoop 2.4
-
- git checkout branch-2
- export HADOOP_VERSION=2.4.0-SNAPSHOT
-
-To build and install it locally, skipping the tests:
-
- mvn clean install -DskipTests
-
-To make a tarball for use in test runs:
-
- #On osx
- mvn clean install package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip=true
-
- # on linux
- mvn clean package -Pdist -Pnative -Dtar -DskipTests -Dmaven.javadoc.skip=true
-
-Then expand this
-
- pushd hadoop-dist/target/
- gunzip hadoop-$HADOOP_VERSION.tar.gz
- tar -xvf hadoop-$HADOOP_VERSION.tar
- popd
-
-This creates an expanded version of Hadoop. You can now actually run Hadoop
-from this directory. Do note that unless you have the native code built for
-your target platform, Hadoop will be slower.
-
-## building a compatible HBase version
-
-If you need to build a version of HBase -rather than use a released version,
-here are the instructions (for the hbase-0.98 release branch)
-
-Checkout the HBase `trunk` branch from apache svn/github.
-
-
- git clone git://git.apache.org/hbase.git
- cd hbase
- git remote rename origin apache
- git fetch --tags apache
-
-then
-
- git checkout -b apache/0.98
-or
-
- git checkout tags/0.98.1
-
-If you have already been building versions of HBase, remove the existing
-set of artifacts for safety:
-
- rm -rf ~/.m2/repository/org/apache/hbase/
-
-The maven command for building hbase artifacts against this hadoop version is
-
- mvn clean install assembly:single -DskipTests -Dmaven.javadoc.skip=true
-
-To use a different version of Hadoop from that defined in the `hadoop-two.version`
-property of`/pom.xml`:
-
- mvn clean install assembly:single -DskipTests -Dmaven.javadoc.skip=true -Dhadoop-two.version=$HADOOP_VERSION
-
-This will create an hbase `tar.gz` file in the directory `hbase-assembly/target/`
-in the hbase source tree.
-
- export HBASE_VERSION=0.98.1
-
- pushd hbase-assembly/target
- gunzip hbase-$HBASE_VERSION-bin.tar.gz
- tar -xvf hbase-$HBASE_VERSION-bin.tar
- gzip hbase-$HBASE_VERSION-bin.tar
- popd
-
-This will create an untarred directory containing
-hbase. Both the `.tar.gz` and untarred file are needed for testing. Most
-tests just work directly with the untarred file as it saves time uploading
-and downloading then expanding the file.
-
-(and if you set `HBASE_VERSION` to something else, you can pick up that version
--making sure that slider is in sync)
-
-For more information (including recommended Maven memory configuration options),
-see [HBase building](http://hbase.apache.org/book/build.html)
-
-For building just the JAR files:
-
- mvn clean install -DskipTests -Dhadoop.profile=2.0 -Dhadoop-two.version=$HADOOP_VERSION
-
-*Tip:* you can force set a version in Maven by having it update all the POMs:
-
- mvn versions:set -DnewVersion=0.98.1-SNAPSHOT
-
-## Building Accumulo
-
-Clone accumulo from apache;
-
- git clone http://git-wip-us.apache.org/repos/asf/accumulo.git
-
-
-Check out branch 1.6.1-SNAPSHOT
-
- git checkout 1.6.1-SNAPSHOT
-
-In the accumulo project directory, build it
-
- mvn clean install -Passemble -DskipTests -Dmaven.javadoc.skip=true \
- -Dhadoop.profile=2
-
-The default Hadoop version for accumulo-1.6.1 is hadoop 2.4.0; to build
-against a different version use the command
-
- mvn clean install -Passemble -DskipTests -Dmaven.javadoc.skip=true \
- -Dhadoop.profile=2 -Dhadoop.version=$HADOOP_VERSION
-
-This creates an accumulo tar.gz file in `assemble/target/`. Extract this
-to create an expanded directory
-
- accumulo/assemble/target/accumulo-1.6.1-SNAPSHOT-bin.tar.gz
-
- This can be done with the command sequence
-
- export ACCUMULO_VERSION=1.6.1-SNAPSHOT
-
- pushd assemble/target/
- gunzip -f accumulo-$ACCUMULO_VERSION-bin.tar.gz
- tar -xvf accumulo-$ACCUMULO_VERSION-bin.tar.gz
- popd
-
-Note that the final location of the accumulo files is needed for the configuration,
-it may be directly under target/ or it may be in a subdirectory, with
-a path such as `target/accumulo-$ACCUMULO_VERSION-dev/accumulo-$ACCUMULO_VERSION/`
-
-
-## Testing
-
-### Configuring Slider to locate the relevant artifacts
-
-You must have the file `src/test/resources/slider-test.xml` (this
-is ignored by git), declaring where HBase, accumulo, Hadoop and zookeeper are:
-
- <configuration>
-
- <property>
- <name>slider.test.hbase.home</name>
- <value>/home/slider/hbase/hbase-assembly/target/hbase-0.98.0-SNAPSHOT</value>
- <description>HBASE Home</description>
- </property>
-
- <property>
- <name>slider.test.hbase.tar</name>
- <value>/home/slider/hbase/hbase-assembly/target/hbase-0.98.0-SNAPSHOT-bin.tar.gz</value>
- <description>HBASE archive URI</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.home</name>
- <value>/home/slider/accumulo/assemble/target/accumulo-1.6.1-SNAPSHOT/</value>
- <description>Accumulo Home</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.tar</name>
- <value>/home/slider/accumulo/assemble/target/accumulo-1.6.1-SNAPSHOT-bin.tar.gz</value>
- <description>Accumulo archive URI</description>
- </property>
-
- <property>
- <name>zk.home</name>
- <value>
- /home/slider/Apps/zookeeper</value>
- <description>Zookeeper home dir on target systems</description>
- </property>
-
- <property>
- <name>hadoop.home</name>
- <value>
- /home/slider/hadoop-common/hadoop-dist/target/hadoop-2.3.0</value>
- <description>Hadoop home dir on target systems</description>
- </property>
-
- </configuration>
-
-
-## Debugging a failing test
-
-1. Locate the directory `target/$TESTNAME` where TESTNAME is the name of the
-test case and or test method. This directory contains the Mini YARN Cluster
-logs. For example, `TestLiveRegionService` stores its data under
-`target/TestLiveRegionService`
-
-1. Look under that directory for `-logdir` directories, then an application
-and container containing logs. There may be more than node being simulated;
-every node manager creates its own logdir.
-
-1. Look for the `out.txt` and `err.txt` files for stdout and stderr log output.
-
-1. Slider uses SLF4J to log to `out.txt`; remotely executed processes may use
-either stream for logging
-
-Example:
-
- target/TestLiveRegionService/TestLiveRegionService-logDir-nm-1_0/application_1376095770244_0001/container_1376095770244_0001_01_000001/out.txt
-
-1. The actual test log from JUnit itself goes to the console and into
-`target/surefire/`; this shows the events happening in the YARN services as well
- as (if configured) HDFS and Zookeeper. It is noisy -everything after the *teardown*
- message happens during cluster teardown, after the test itself has been completed.
- Exceptions and messages here can generally be ignored.
-
-This is all a bit complicated -debugging is simpler if a single test is run at a
-time, which is straightforward
-
- mvn clean test -Dtest=TestLiveRegionService
-
-
-### Building the JAR file
-
-You can create the JAR file and set up its directories with
-
- mvn package -DskipTests
-
-# Development Notes
-
-<!---
-## Git branch model
-
-
-The git branch model uses is
-[Git Flow](http://nvie.com/posts/a-successful-git-branching-model/).
-
-This is a common workflow model for Git, and built in to
-[Atlassian Source Tree](http://sourcetreeapp.com/).
-
-The command line `git-flow` tool is easy to install
-
- brew install git-flow
-
-or
-
- apt-get install git-flow
-
-You should then work on all significant features in their own branch and
-merge them back in when they are ready.
-
-
- # until we get a public JIRA we're just using an in-house one. sorry
- git flow feature start BUG-8192
-
- # finishes merges back in to develop/
- git flow feature finish BUG-8192
-
- # release branch
- git flow release start 0.4.0
-
- git flow release finish 0.4.0
--->
-
-## Attention OS/X developers
-
-YARN on OS/X doesn't terminate subprocesses the way it does on Linux, so
-HBase Region Servers created by the hbase shell script remain running
-even after the tests terminate.
-
-This causes some tests -especially those related to flexing down- to fail,
-and test reruns may be very confused. If ever a test fails because there
-are too many region servers running, this is the likely cause
-
-After every test run: do a `jps -v` to look for any leftover HBase services
--and kill them.
-
-Here is a handy bash command to do this
-
- jps -l | grep HRegion | awk '{print $1}' | xargs kill -9
-
-
-## Groovy
-
-Slider uses Groovy 2.x as its language for writing tests -for better assertions
-and easier handling of lists and closures. Although the first prototype
-used Groovy on the production source, this was dropped in favor of
-a Java-only production codebase.
-
-## Maven utils
-
-
-Here are some handy aliases to make maven easier
-
- alias mci='mvn clean install -DskipTests'
- alias mi='mvn install -DskipTests'
- alias mvct='mvn clean test'
- alias mvnsite='mvn site:site -Dmaven.javadoc.skip=true'
- alias mvt='mvn test'
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/developing/functional_tests.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/developing/functional_tests.md b/src/site/markdown/developing/functional_tests.md
deleted file mode 100644
index 8b5e170..0000000
--- a/src/site/markdown/developing/functional_tests.md
+++ /dev/null
@@ -1,416 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Testing Apache Slider
-
- The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
- NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
- "OPTIONAL" in this document are to be interpreted as described in
- RFC 2119.
-
-# Functional Tests
-
-The functional test suite is designed to test slider against
-a live cluster.
-
-For these to work you need
-
-1. A YARN Cluster -secure or insecure
-1. A `slider-client.xml` file configured to interact with the cluster
-1. Agent
-1. HBase tests: HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
-directory
-1. Accumulo Tests Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
-directory
-
-## Configuration of functional tests
-
-Maven needs to be given
-
-1. A path to the expanded test archive
-1. A path to a slider configuration directory for the cluster
-
-The path for the expanded test is automatically calculated as being the directory under
-`..\slider-assembly\target` where an untarred slider distribution can be found.
-If it is not present, the tests will fail
-
-The path to the configuration directory must be supplied in the property
-`slider.conf.dir` which can be set on the command line
-
- mvn test -Dslider.conf.dir=src/test/configs/sandbox/slider
-
-It can also be set in the (optional) file `slider-funtest/build.properties`:
-
- slider.conf.dir=src/test/configs/sandbox/slider
-
-This file is loaded whenever a slider build or test run takes place
-
-## Configuration of `slider-client.xml`
-
-The `slider-client.xml` must have extra configuration options for both the HBase and
-Accumulo tests, as well as a common set for actually talking to a YARN cluster.
-
-## Disabling the functional tests entirely
-
-All functional tests which require a live YARN cluster
-can be disabled through the property `slider.funtest.enabled`
-
- <property>
- <name>slider.funtest.enabled</name>
- <value>false</value>
- </property>
-
-There is a configuration do do exactly this in
-`src/test/configs/offline/slider`:
-
- slider.conf.dir=src/test/configs/offline/slider
-
-Tests which do not require a live YARN cluster will still run;
-these verify that the `bin/slider` script works.
-
-### Non-mandatory options
-
-The following test options may be added to `slider-client.xml` if the defaults
-need to be changed
-
- <property>
- <name>slider.test.zkhosts</name>
- <description>comma separated list of ZK hosts</description>
- <value>localhost</value>
- </property>
-
- <property>
- <name>slider.test.thaw.wait.seconds</name>
- <description>Time to wait in seconds for a thaw to result in a running AM</description>
- <value>60000</value>
- </property>
-
- <property>
- <name>slider.test.freeze.wait.seconds</name>
- <description>Time to wait in seconds for a freeze to halt the cluster</description>
- <value>60000</value>
- </property>
-
- <property>
- <name>slider.test.timeout.millisec</name>
- <description>Time out in milliseconds before a test is considered to have failed.
- There are some maven properties which also define limits and may need adjusting</description>
- <value>180000</value>
- </property>
-
- <property>
- <name>slider.test.yarn.ram</name>
- <description>Size in MB to ask for containers</description>
- <value>192</value>
- </property>
-
-
-Note that while the same properties need to be set in
-`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
-filesystem -here a URI to a path visible across all nodes in the cluster are required
-the tests do not copy the .tar/.tar.gz files over. The application configuration
-directories may be local or remote -they are copied into the `.slider` directory
-during cluster creation.
-
-## Provider-specific parameters
-
-An individual provider can pick up settings from their own
-`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
-We strongly advice placing all the values in the `slider-core` file.
-
-1. All uncertainty about which file is picked up on the class path first goes
-away
-2. There's one place to keep all the configuration values in sync.
-
-### Agent Tests
-
-Agent tests are executed through the following mvn command executed at slider/slider-funtest:
-
-```
-cd slider-funtest
-mvn test -Dslider.conf.dir=../src/test/clusters/remote/slider -Dtest=TestAppsThroughAgent -DfailIfNoTests=false
-```
-
-**Enable/Execute the tests**
-
-To enable the test ensure that *slider.test.agent.enabled* is set to *true*.
-
- <property>
- <name>slider.test.agent.enabled</name>
- <description>Flag to enable/disable Agent tests</description>
- <value>true</value>
- </property>
-
-**Test setup**
-
-Edit config file src/test/clusters/remote/slider/slider-client.xml and ensure that the host names are accurate for the test cluster.
-
-**User setup**
-
-Ensure that the user, running the test, is present on the cluster against which you are running the tests. The user must be a member of the hadoop group.
-
-E.g. adduser **testuser** -d /home/**testuser** -G hadoop -m
-
-**HDFS Setup**
-
-Set up hdfs folders for slider and test user
-
-* su hdfs
-* hdfs dfs -mkdir /slider
-* hdfs dfs -chown testuser:hdfs /slider
-* hdfs dfs -mkdir /user/testuser
-* hdfs dfs -chown testuser:hdfs /user/testuser
-
-Load up agent package and config
-
-* su **testuser**
-* hdfs dfs -mkdir /slider/agent
-* hdfs dfs -mkdir /slider/agent/conf
-* hdfs dfs -copyFromLocal SLIDER_INSTALL_LOC/agent/conf/agent.ini /slider/agent/conf
-
-Ensure correct host name is provided for the agent tarball.
-
- <property>
- <name>slider.test.agent.tar</name>
- <description>Path to the Agent Tar file in HDFS</description>
- <value>hdfs://NN_HOSTNAME:8020/slider/agent/slider-agent.tar.gz</value>
- </property>
-
-
-
-### HBase Tests
-
-The HBase tests can be enabled or disabled
-
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
-Mandatory test parameters must be added to `slider-client.xml`
-
- <property>
- <name>slider.test.hbase.tar</name>
- <description>Path to the HBase Tar file in HDFS</description>
- <value>hdfs://sandbox:8020/user/slider/hbase.tar.gz</value>
- </property>
-
- <property>
- <name>slider.test.hbase.appconf</name>
- <description>Path to the directory containing the HBase application config</description>
- <value>file://${user.dir}/src/test/configs/sandbox/hbase</value>
- </property>
-
-Optional parameters:
-
- <property>
- <name>slider.test.hbase.launch.wait.seconds</name>
- <description>Time to wait in seconds for HBase to start</description>
- <value>1800</value>
- </property>
-
-#### Accumulo configuration options
-
-Enable/disable the tests
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
-Optional parameters
-
- <property>
- <name>slider.test.accumulo.launch.wait.seconds</name>
- <description>Time to wait in seconds for Accumulo to start</description>
- <value>1800</value>
- </property>
-
-### Configuring the YARN cluster for tests
-
-Here are the configuration options we use in `yarn-site.xml` for testing:
-
-These tell YARN to ignore memory requirements in allocating VMs, and
-to keep the log files around after an application run.
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1</value>
- </property>
- <property>
- <description>Whether physical memory limits will be enforced for
- containers.
- </description>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
- <!-- we really don't want checking here-->
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <!-- how long after a failure to see what is left in the directory-->
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>60000</value>
- </property>
-
- <!--ten seconds before the process gets a -9 -->
- <property>
- <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
- <value>30000</value>
- </property>
-
-
-## Testing against a secure cluster
-
-To test against a secure cluster
-
-1. `slider-client.xml` must be configured as per [Security](../security.html).
-1. the client must have the kerberos tokens issued so that the user running
-the tests has access to HDFS and YARN.
-
-If there are problems authenticating (including the cluster being offline)
-the tests appear to hang
-
-### Validating the configuration
-
- mvn test -Dtest=TestBuildSetup
-
-### Using relative paths in test configurations
-
-When you are sharing configurations across machines via SCM or similar,
-its impossible to have absolute paths in the configuration options to
-the location of items in the local filesystem (e.g. configuration directories).
-
-There's two techniques
-
-1. Keep the data in HDFS and refer to it there. This works if there is a shared,
-persistent HDFS cluster.
-
-1. Use the special property `slider.test.conf.dir` that is set to the path
-of the directory, and which can then be used to create an absolute path
-from paths relative to the configuration dir:
-
- <property>
- <name>slider.test.hbase.appconf</name>
- <description>Path to the directory containing the HBase application config</description>
- <value>file://${slider.test.conf.dir}/../hbase</value>
- </property>
-
-
-If the actual XML file path is required, a similar property
-`slider.test.conf.xml` is set.
-
-
-## Parallel execution
-
-Attempts to run test cases in parallel failed -even with a configuration
-to run methods in a class sequentially, but separate classes independently.
-
-Even after identifying and eliminating some unintended sharing of static
-mutable variables, trying to run test cases in parallel seemed to hang
-tests and produce timeouts.
-
-For this reason parallel tests have been disabled. To accelerate test runs
-through parallelization, run different tests on different hosts instead.
-
-## Other constraints
-
-* Port assignments SHOULD NOT be fixed, as this will cause clusters to fail if
-there are too many instances of a role on a same host, or if other tests are
-using the same port.
-* If a test does need to fix a port, it MUST be for a single instance of a role,
-and it must be different from all others. The assignment should be set in
-`org.apache.slider.funtest.itest.PortAssignments` so as to ensure uniqueness
-over time. Otherwise: use the value of `0` to allow the OS to assign free ports
-on demand.
-
-## Test Requirements
-
-
-1. Test cases should be written so that each class works with exactly one
-Slider-deployed cluster
-1. Every test MUST have its own cluster name -preferably derived from the
-classname.
-1. This cluster should be deployed in an `@BeforeClass` method.
-1. The `@AfterClass` method MUST tear this cluster down.
-1. Tests must skip their execution if functional tests -or the
-specific hbase or accumulo categories- are disabled.
-1. Tests within the suite (i.e. class) must be designed to be independent
--to work irrespectively of the ordering of other tests.
-
-## Running and debugging the functional tests.
-
-The functional tests all
-
-1. In the root `slider` directory, build a complete Slider release
-
- mvn install -DskipTests
-1. Start the YARN cluster/set up proxies to connect to it, etc.
-
-1. In the `slider-funtest` dir, run the tests
-
- mvn test
-
-A common mistake during development is to rebuild the `slider-core` JARs
-then the `slider-funtest` tests without rebuilding the `slider-assembly`.
-In this situation, the tests are in sync with the latest build of the code
--including any bug fixes- but the scripts executed by those tests are
-of a previous build of `slider-core.jar`. As a result, the fixes are not picked
-up.
-
-#### To propagate changes in slider-core through to the funtest classes for
-testing, you must build/install all the slider packages from the root assembly.
-
- mvn clean install -DskipTests
-
-## Limitations of slider-funtest
-
-1. All tests run from a single client -workload can't scale
-1. Output from failed AM and containers aren't collected
-
-## Troubleshooting the functional tests
-
-1. If application instances fail to come up as there are still outstanding
-requests, it means that YARN didn't have the RAM/cores to spare for the number
-of containers. Edit the `slider.test.yarn.ram` to make it smaller.
-
-1. If you are testing in a local VM and stops responding, it'll have been
-swapped out to RAM. Rebooting can help, but for a long term fix go through
-all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
-smaller numbers, like 256M each. Also: turn off unused services (hcat, oozie,
-webHDFS)
-
-1. The YARN UI will list the cluster launches -look for the one
-with a name close to the test and view its logs
-
-1. Container logs will appear "elsewhere". The log lists
-the containers used -you may be able to track the logs
-down from the specific nodes.
-
-1. If you browse the filesystem, look for the specific test clusters
-in `~/.slider/cluster/$testname`
-
-1. If you are using a secure cluster, make sure that the clocks
-are synchronized, and that you have a current token -`klist` will
-tell you this. In a VM: install and enable `ntp`, consider rebooting if ther
-are any problems. Check also that it has the same time zone settings
-as the host OS.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/developing/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/developing/index.md b/src/site/markdown/developing/index.md
deleted file mode 100644
index 4372495..0000000
--- a/src/site/markdown/developing/index.md
+++ /dev/null
@@ -1,35 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Developing Apache Slider
-
-Slider is an open source project -anyone is free to contributed, and we
-strongly encourage people to do so.
-
-Here are documents covering how to go about building, testing and releasing
-Slider
-
-* [Building](building.html)
-* [Debugging](../debugging.html)
-* [Testing](testing.html)
-* [Functional Testing](functional_tests.html)
-* [Manual Testing](manual_testing.html)
-* [Agent test setup](agent_test_setup.html)
-* [Releasing](releasing.html)
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/developing/manual_testing.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/developing/manual_testing.md b/src/site/markdown/developing/manual_testing.md
deleted file mode 100644
index bfc7e7c..0000000
--- a/src/site/markdown/developing/manual_testing.md
+++ /dev/null
@@ -1,53 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Manually Testing Apache Slider
-
-Manual testing invloves using Slider package and an AppPackage to perform basic
- cluster functionalities such as create/destroy, flex up/down, and freeze/thaw.
- A python helper script is provided that can be used to automatically test and app package.
-
-## `SliderTester.py`
-Details to be added.
-
-## `SliderTester.ini`
-The various config parameters are:
-
-### slider
-* `package`: location of the slider package
-* `jdk.path`: jdk path on the test hosts
-
-### app
-* `package`: location of the app package
-
-### cluster
-* `yarn.application.classpath`: yarn application classpaths
-* `slider.zookeeper.quorum`: the ZK quorum hosts
-* `yarn.resourcemanager.address`:
-* `yarn.resourcemanager.scheduler.address`:
-* `fs.defaultFS`: e.g. `hdfs://NN_HOST:8020`
-
-### test
-* `app.user`: user to use for app creation
-* `hdfs.root.user`: hdfs root user
-* `hdfs.root.dir`: HDFS root, default /slidertst
-* `hdfs.user.dir`: HDFS user dir, default /user
-* `test.root`: local test root folder, default /test
-* `cluster.name`: name of the test cluster, default tst1
-* `cluster.type`: cluster type to build and test, e.g. hbase,storm,accumulo
-
-### agent
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/developing/releasing.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/developing/releasing.md b/src/site/markdown/developing/releasing.md
deleted file mode 100644
index a8380bf..0000000
--- a/src/site/markdown/developing/releasing.md
+++ /dev/null
@@ -1,195 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-
-# Apache Slider Release Process
-
-Here is our release process.
-
-
-## IMPORTANT: THIS IS OUT OF DATE WITH THE MOVE TO THE ASF ##
-
-### Before you begin
-
-Check out the latest version of the develop branch,
-run the tests. This should be done on a checked out
-version of the code that is not the one you are developing on
-(ideally, a clean VM), to ensure that you aren't releasing a slightly
-modified version of your own, and that you haven't accidentally
-included passwords or other test run details into the build resource
-tree.
-
-The `slider-funtest` functional test package is used to run functional
-tests against a running Hadoop YARN cluster. It needs to be configured
-according to the instructions in [testing](testing.html) to
-create HBase and Accumulo clusters in the YARN cluster.
-
-*Make sure that the functional tests are passing (and not being skipped) before
-starting to make a release*
-
-
-
-**Step #1:** Create a JIRA for the release, estimate 3h
-(so you don't try to skip the tests)
-
- export SLIDER_RELEASE_JIRA=SLIDER-13927
-
-**Step #2:** Check everything in. Git flow won't let you progress without this.
-
-**Step #3:** Git flow: create a release branch
-
- export SLIDER_RELEASE=0.5.2
-
- git flow release start slider-$SLIDER_RELEASE
-
-**Step #4:** in the new branch, increment those version numbers using (the maven
-versions plugin)[http://mojo.codehaus.org/versions-maven-plugin/]
-
- mvn versions:set -DnewVersion=$SLIDER_RELEASE
-
-
-**Step #5:** commit the changed POM files
-
- git add <changed files>
- git commit -m "$SLIDER_RELEASE_JIRA updating release POMs for $SLIDER_RELEASE"
-
-
-**Step #6:** Do a final test run to make sure nothing is broken
-
-In the `slider` directory, run:
-
- mvn clean install -DskipTests
-
-Once everything is built- including .tar files, run the tests
-
- mvn test
-
-This will run the functional tests as well as the `slider-core` tests.
-
-It is wise to reset any VMs here, and on live clusters kill all running jobs.
-This stops functional tests failing because the job doesn't get started before
-the tests time out.
-
-As the test run takes 30-60+ minutes, now is a good time to consider
-finalizing the release notes.
-
-
-**Step #7:** Build the release package
-
-Run
-
- mvn clean site:site site:stage package -DskipTests
-
-
-
-**Step #8:** validate the tar file
-
-Look in `slider-assembly/target` to find the `.tar.gz` file, and the
-expanded version of it. Inspect that expanded version to make sure that
-everything looks good -and that the versions of all the dependent artifacts
-look good too: there must be no `-SNAPSHOT` dependencies.
-
-
-**Step #9:** Build the release notes
-
-Create a a one-line plain text release note for commits and tags
-And a multi-line markdown release note, which will be used for artifacts.
-
-
- Release against hadoop 2.4.0, HBase-0.98.1 and Accumulo 1.5.1 artifacts.
-
-The multi-line release notes should go into `slider/src/site/markdown/release_notes`.
-
-
-These should be committed
-
- git add --all
- git commit -m "$SLIDER_RELEASE_JIRA updating release notes"
-
-**Step #10:** End the git flow
-
-Finish the git flow release, either in the SourceTree GUI or
-the command line:
-
-
- git flow release finish slider-$SLIDER_RELEASE
-
-
-On the command line you have to enter the one-line release description
-prepared earlier.
-
-You will now be back on the `develop` branch.
-
-**Step #11:** update mvn versions
-
-Switch back to `develop` and update its version number past
-the release number
-
-
- export SLIDER_RELEASE=0.6.0-SNAPSHOT
- mvn versions:set -DnewVersion=$SLIDER_RELEASE
- git commit -a -m "$SLIDER_RELEASE_JIRA updating development POMs to $SLIDER_RELEASE"
-
-**Step #12:** Push the release and develop branches to github
-
- git push origin master develop
-
-(assuming that `origin` maps to `git@github.com:hortonworks/slider.git`;
- you can check this with `git remote -v`
-
-
-The `git-flow` program automatically pushes up the `release/slider-X.Y` branch,
-before deleting it locally.
-
-If you are planning on any release work of more than a single test run,
-consider having your local release branch track the master.
-
-
-**Step #13:** ### Release on github small artifacts
-
-Browse to https://github.com/hortonworks/slider/releases/new
-
-Create a new release on the site by following the instructions
-
-Files under 5GB can be published directly. Otherwise, follow step 14
-
-**Step #14:** For releasing via an external CDN (e.g. Rackspace Cloud)
-
-Using the web GUI for your particular distribution network, upload the
-`.tar.gz` artifact
-
-After doing this, edit the release notes on github to point to the
-tar file's URL.
-
-Example:
- [Download slider-0.10.1-all.tar.gz](http://dffeaef8882d088c28ff-185c1feb8a981dddd593a05bb55b67aa.r18.cf1.rackcdn.com/slider-0.10.1-all.tar.gz)
-
-**Step #15:** Announce the release
-
-**Step #16:** Finish the JIRA
-
-Log the time, close the issue. This should normally be the end of a
-sprint -so wrap that up too.
-
-**Step #17:** Get back to developing!
-
-Check out the develop branch and purge all release artifacts
-
- git checkout develop
- git pull origin
- mvn clean
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/developing/testing.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/developing/testing.md b/src/site/markdown/developing/testing.md
deleted file mode 100644
index 2c2ae62..0000000
--- a/src/site/markdown/developing/testing.md
+++ /dev/null
@@ -1,182 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Testing Apache Slider
-
- The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
- NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
- "OPTIONAL" in this document are to be interpreted as described in
- RFC 2119.
-
-## Standalone Tests
-
-Slider core contains a suite of tests that are designed to run on the local machine,
-using Hadoop's `MiniDFSCluster` and `MiniYARNCluster` classes to create small,
-one-node test clusters. All the YARN/HDFS code runs in the JUnit process; the
-AM and spawned processeses run independently.
-
-
-
-### For HBase Tests in `slider-providers/hbase`
-
-Requirements
-* A copy of `hbase.tar.gz` in the local filesystem
-* A an expanded `hbase.tar.gz` in the local filesystem
-
-
-### For Accumulo Tests in `slider-providers/accumulo`
-* A copy of `accumulo.tar.gz` in the local filesystem,
-* An expanded `accumulo.tar.gz` in the local filesystem,
-* an expanded Zookeeper installation
-
-All of these need to be defined in the file `slider-core/src/test/resources/slider-test.xml`
-
-Example:
-
- <configuration>
-
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.hbase.home</name>
- <value>/home/slider/hbase-0.98.0</value>
- <description>HBASE Home</description>
- </property>
-
- <property>
- <name>slider.test.hbase.tar</name>
- <value>/home/slider/Projects/hbase-0.98.0-bin.tar.gz</value>
- <description>HBASE archive URI</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.home</name>
- <value>
- /home/slider/accumulo-1.6.0-SNAPSHOT/</value>
- <description>Accumulo Home</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.tar</name>
- <value>/home/slider/accumulo-1.6.0-SNAPSHOT-bin.tar</value>
- <description>Accumulo archive URI</description>
- </property>
-
- <property>
- <name>slider.test.am.restart.time</name>
- <description>Time in millis to await an AM restart</description>
- <value>30000</value>
- </property>
-
- <property>
- <name>zk.home</name>
- <value>/home/slider/zookeeper</value>
- <description>Zookeeper home dir on target systems</description>
- </property>
-
- <property>
- <name>hadoop.home</name>
- <value>/home/slider/hadoop-2.2.0</value>
- <description>Hadoop home dir on target systems</description>
- </property>
-
- </configuration>
-
-*Important:* For the local tests, a simple local filesystem path is used for
-all the values.
-
-For the functional tests, the accumulo and hbase tar properties will
-need to be set to a URL of a tar file that is accessible to all the
-nodes in the cluster -which usually means HDFS, and so an `hdfs://` URL
-
-
-## Provider-specific parameters
-
-An individual provider can pick up settings from their own
-`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
-We strongly advice placing all the values in the `slider-core` file.
-
-1. All uncertainty about which file is picked up on the class path first goes
-away
-2. There's one place to keep all the configuration values in sync.
-
-### Agent Tests
-
-
-### HBase Tests
-
-The HBase tests can be enabled or disabled
-
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
-Mandatory test parameters must be added to `slider-client.xml`
-
-
- <property>
- <name>slider.test.hbase.tar</name>
- <description>Path to the HBase Tar file in HDFS</description>
- <value>hdfs://sandbox:8020/user/slider/hbase.tar.gz</value>
- </property>
-
- <property>
- <name>slider.test.hbase.appconf</name>
- <description>Path to the directory containing the HBase application config</description>
- <value>file://${user.dir}/src/test/configs/sandbox/hbase</value>
- </property>
-
-Optional parameters:
-
- <property>
- <name>slider.test.hbase.launch.wait.seconds</name>
- <description>Time to wait in seconds for HBase to start</description>
- <value>180000</value>
- </property>
-
-
-#### Accumulo configuration options
-
-Enable/disable the tests
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
-
-Optional parameters
-
- <property>
- <name>slider.test.accumulo.launch.wait.seconds</name>
- <description>Time to wait in seconds for Accumulo to start</description>
- <value>180000</value>
- </property>
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/examples.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/examples.md b/src/site/markdown/examples.md
deleted file mode 100644
index f706f2f..0000000
--- a/src/site/markdown/examples.md
+++ /dev/null
@@ -1,159 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Examples
-
-
-## Setup
-
-### Setting up a YARN cluster
-
-For simple local demos, a Hadoop pseudo-distributed cluster will suffice -if on a VM then
-its configuration should be changed to use a public (machine public) IP.
-
-# The examples below all assume there is a cluster node called 'master', which
-hosts the HDFS NameNode and the YARN Resource Manager
-
-
-# preamble
-
- export HADOOP_CONF_DIR=~/conf
- export PATH=~/hadoop/bin:/~/hadoop/sbin:~/zookeeper-3.4.5/bin:$PATH
-
- hdfs namenode -format master
-
-
-
-
-# start all the services
-
- nohup hdfs --config $HADOOP_CONF_DIR namenode &
- nohup hdfs --config $HADOOP_CONF_DIR datanode &
-
-
- nohup yarn --config $HADOOP_CONF_DIR resourcemanager &
- nohup yarn --config $HADOOP_CONF_DIR nodemanager &
-
-# using hadoop/sbin service launchers
-
- hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode
- hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode
- yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
- yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
-
- ~/zookeeper/bin/zkServer.sh start
-
-
-# stop them
-
- hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop namenode
- hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode
-
- yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
- yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
-
-
-
-NN up on [http://master:50070/dfshealth.jsp](http://master:50070/dfshealth.jsp)
-RM yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
-
- ~/zookeeper/bin/zkServer.sh start
-
-
- # shutdown
- ~/zookeeper/bin/zkServer.sh stop
-
-
-Tip: after a successful run on a local cluster, do a quick `rm -rf $HADOOP_HOME/logs`
-to keep the log bloat under control.
-
-## get hbase in
-
-copy to local
-
- get hbase-0.98.0-bin.tar on
-
-
- hdfs dfs -rm hdfs://master:9090/hbase.tar
- hdfs dfs -copyFromLocal hbase-0.98.0-bin.tar hdfs://master:9090/hbase.tar
-
-or
-
- hdfs dfs -copyFromLocal hbase-0.96.0-bin.tar hdfs://master:9090/hbase.tar
- hdfs dfs -ls hdfs://master:9090/
-
-
-### Optional: point bin/slider at your chosen cluster configuration
-
-export SLIDER_CONF_DIR=~/Projects/slider/slider-core/src/test/configs/ubuntu-secure/slider
-
-## Optional: Clean up any existing slider cluster details
-
-This is for demos only, otherwise you lose the clusters and their databases.
-
- hdfs dfs -rm -r hdfs://master:9090/user/home/stevel/.slider
-
-## Create a Slider Cluster
-
-
- slider create cl1 \
- --component worker 1 --component master 1 \
- --manager master:8032 --filesystem hdfs://master:9090 \
- --zkhosts localhost:2181 --image hdfs://master:9090/hbase.tar
-
- # create the cluster
-
- slider create cl1 \
- --component worker 4 --component master 1 \
- --manager master:8032 --filesystem hdfs://master:9090 --zkhosts localhost \
- --image hdfs://master:9090/hbase.tar \
- --appconf file:////Users/slider/Hadoop/configs/master/hbase \
- --compopt master jvm.heap 128 \
- --compopt master env.MALLOC_ARENA_MAX 4 \
- --compopt worker jvm.heap 128
-
- # freeze the cluster
- slider freeze cl1 \
- --manager master:8032 --filesystem hdfs://master:9090
-
- # thaw a cluster
- slider thaw cl1 \
- --manager master:8032 --filesystem hdfs://master:9090
-
- # destroy the cluster
- slider destroy cl1 \
- --manager master:8032 --filesystem hdfs://master:9090
-
- # list clusters
- slider list cl1 \
- --manager master:8032 --filesystem hdfs://master:9090
-
- slider flex cl1 --component worker 2
- --manager master:8032 --filesystem hdfs://master:9090 \
- --component worker 5
-
-## Create an Accumulo Cluster
-
- slider create accl1 --provider accumulo \
- --component master 1 --component tserver 1 --component gc 1 --component monitor 1 --component tracer 1 \
- --manager localhost:8032 --filesystem hdfs://localhost:9000 \
- --zkhosts localhost:2181 --zkpath /local/zookeeper \
- --image hdfs://localhost:9000/user/username/accumulo-1.6.0-SNAPSHOT-bin.tar \
- --appconf hdfs://localhost:9000/user/username/accumulo-conf \
- -O zk.home /local/zookeeper -O hadoop.home /local/hadoop \
- -O site.monitor.port.client 50095 -O accumulo.password secret
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/exitcodes.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/exitcodes.md b/src/site/markdown/exitcodes.md
deleted file mode 100644
index ac63fe1..0000000
--- a/src/site/markdown/exitcodes.md
+++ /dev/null
@@ -1,161 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider Client Exit Codes
-
-Here are the exit codes returned
-
-Exit code values 1 and 2 are interpreted by YARN -in particular converting the
-"1" value from an error into a successful shut down. Slider
-converts the -1 error code from a forked process into `EXIT_MASTER_PROCESS_FAILED`;
-no. 72.
-
-
- /**
- * 0: success
- */
- int EXIT_SUCCESS = 0;
-
- /**
- * -1: generic "false" response. The operation worked but
- * the result was not true
- */
- int EXIT_FALSE = -1;
-
- /**
- * Exit code when a client requested service termination:
- */
- int EXIT_CLIENT_INITIATED_SHUTDOWN = 1;
-
- /**
- * Exit code when targets could not be launched:
- */
- int EXIT_TASK_LAUNCH_FAILURE = 2;
-
- /**
- * Exit code when an exception was thrown from the service:
- */
- int EXIT_EXCEPTION_THROWN = 32;
-
- /**
- * Exit code when a usage message was printed:
- */
- int EXIT_USAGE = 33;
-
- /**
- * Exit code when something happened but we can't be specific:
- */
- int EXIT_OTHER_FAILURE = 34;
-
- /**
- * Exit code when a control-C, kill -3, signal was picked up:
- */
-
- int EXIT_INTERRUPTED = 35;
-
- /**
- * Exit code when the command line doesn't parse:, or
- * when it is otherwise invalid.
- */
- int EXIT_COMMAND_ARGUMENT_ERROR = 36;
-
- /**
- * Exit code when the configurations in valid/incomplete:
- */
- int EXIT_BAD_CONFIGURATION = 37;
-
- /**
- * Exit code when the configurations in valid/incomplete:
- */
- int EXIT_CONNECTIVTY_PROBLEM = 38;
-
- /**
- * internal error: {@value}
- */
- int EXIT_INTERNAL_ERROR = 64;
-
- /**
- * Unimplemented feature: {@value}
- */
- int EXIT_UNIMPLEMENTED = 65;
-
- /**
- * service entered the failed state: {@value}
- */
- int EXIT_YARN_SERVICE_FAILED = 66;
-
- /**
- * service was killed: {@value}
- */
- int EXIT_YARN_SERVICE_KILLED = 67;
-
- /**
- * timeout on monitoring client: {@value}
- */
- int EXIT_TIMED_OUT = 68;
-
- /**
- * service finished with an error: {@value}
- */
- int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 69;
-
- /**
- * the application instance is unknown: {@value}
- */
- int EXIT_UNKNOWN_INSTANCE = 70;
-
- /**
- * the application instance is in the wrong state for that operation: {@value}
- */
- int EXIT_BAD_STATE = 71;
-
- /**
- * A spawned master process failed
- */
- int EXIT_PROCESS_FAILED = 72;
-
- /**
- * The cluster failed -too many containers were
- * failing or some other threshold was reached
- */
- int EXIT_DEPLOYMENT_FAILED = 73;
-
- /**
- * The application is live -and the requested operation
- * does not work if the cluster is running
- */
- int EXIT_APPLICATION_IN_USE = 74;
-
- /**
- * There already is an application instance of that name
- * when an attempt is made to create a new instance
- */
- int EXIT_INSTANCE_EXISTS = 75;
-
- /**
- * The resource was not found
- */
- int EXIT_NOT_FOUND = 77;
-
-## Other exit codes
-
-YARN itself can fail containers, here are some of the causes we've seen
-
-
- 143: Appears to be triggered by the container exceeding its cgroup memory
- limits
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/getting_started.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/getting_started.md b/src/site/markdown/getting_started.md
deleted file mode 100644
index 7488c6c..0000000
--- a/src/site/markdown/getting_started.md
+++ /dev/null
@@ -1,509 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# Apache Slider: Getting Started
-
-
-## Introduction
-
-The following provides the steps required for setting up a cluster and deploying a YARN hosted application using Slider.
-
-* [Prerequisites](#sysreqs)
-
-* [Setup the Cluster](#setup)
-
-* [Download Slider Packages](#download)
-
-* [Build Slider](#build)
-
-* [Install Slider](#install)
-
-* [Deploy Slider Resources](#deploy)
-
-* [Download Sample Application Packages](#downsample)
-
-* [Install, Configure, Start and Verify Sample Application](#installapp)
-
-* [Appendix A: Storm Sample Application Specifications](#appendixa)
-
-* [Appendix B: HBase Sample Application Specifications](#appendixb)
-
-## <a name="sysreqs"></a>System Requirements
-
-The Slider deployment has the following minimum system requirements:
-
-* Hadoop 2.4+
-
-* Required Services: HDFS, YARN, MapReduce2 and ZooKeeper
-
-* Oracle JDK 1.7 (64-bit)
-
-## <a name="setup"></a>Setup the Cluster
-
-After setting up your Hadoop cluster (using Ambari or other means) with the
-services listed above, modify your YARN configuration to allow for multiple
-containers on a single host. In `yarn-site.xml` make the following modifications:
-
-<table>
- <tr>
- <td>Property</td>
- <td>Value</td>
- </tr>
- <tr>
- <td>yarn.scheduler.minimum-allocation-mb</td>
- <td>>= 256</td>
- </tr>
- <tr>
- <td>yarn.nodemanager.delete.debug-delay-sec</td>
- <td>>= 3600 (to retain for an hour)</td>
- </tr>
-</table>
-
-
-There are other options detailed in the Troubleshooting file available <a href="troubleshooting.html">here</a>
-
-
-## <a name="download"></a>Download Slider Packages
-
-Slider releases are available at
-[https://www.apache.org/dyn/closer.cgi/incubator/slider](https://www.apache.org/dyn/closer.cgi/incubator/slider).
-
-## <a name="build"></a>Build Slider
-
-* From the top level directory, execute `mvn clean install -DskipTests`
-* Use the generated compressed tar file in slider-assembly/target directory (e.g. slider-0.30.0-all.tar.gz) for the subsequent steps
-
-## <a name="install"></a>Install Slider
-
-Follow the following steps to expand/install Slider:
-
- mkdir ${slider-install-dir*;
-
- cd ${slider-install-dir}
-
-Login as the "yarn" user (assuming this is a host associated with the installed cluster). E.g., `su yarn`
-*This assumes that all apps are being run as ‘yarn’ user. Any other user can be used to run the apps - ensure that file permission is granted as required.*
-
-Expand the tar file: `tar -xvf slider-0.30.0-all.tar.gz`
-
-Browse to the Slider directory: `cd slider-0.30.0/bin`
-
- export PATH=$PATH:/usr/jdk64/jdk1.7.0_45/bin
-
-(or the path to the JDK bin directory)
-
-Modify Slider configuration file `${slider-install-dir}/slider-0.30.0/conf/slider-client.xml` to add the following properties:
-
- <property>
- <name>yarn.application.classpath</name>
- <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
- </property>
-
- <property>
- <name>slider.zookeeper.quorum</name>
- <value>yourZooKeeperHost:port</value>
- </property>
-
-
-In addition, specify the scheduler and HDFS addresses as follows:
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>yourResourceManagerHost:8050</value>
- </property>
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>yourResourceManagerHost:8030</value>
- </property>
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://yourNameNodeHost:8020</value>
- </property>
-
-
-Execute:
-
- ${slider-install-dir}/slider-0.30.0/bin/slider version
-
-Ensure there are no errors and you can see "Compiled against Hadoop 2.4.0"
-
-## <a name="deploy"></a>Deploy Slider Resources
-
-Ensure that all file folders are accessible to the user creating the application instance. The example assumes "yarn" to be that user.
-
-### Create HDFS root folder for Slider
-
-Perform the following steps to create the Slider root folder with the appropriate permissions:
-
- su hdfs
-
- hdfs dfs -mkdir /slider
-
- hdfs dfs -chown yarn:hdfs /slider
-
- hdfs dfs -mkdir /user/yarn
-
- hdfs dfs -chown yarn:hdfs /user/yarn
-
-### Load Slider Agent
-
- su yarn
-
- hdfs dfs -mkdir /slider/agent
-
- hdfs dfs -mkdir /slider/agent/conf
-
- hdfs dfs -copyFromLocal ${slider-install-dir}/slider-0.30.0/agent/slider-agent-0.30.0.tar.gz /slider/agent
-
-### Create and deploy Slider Agent configuration
-
-Create an agent config file (agent.ini) based on the sample available at:
-
- ${slider-install-dir}/slider-0.30.0/agent/conf/agent.ini
-
-The sample agent.ini file can be used as is (see below). Some of the parameters of interest are:
-
-# `log_level` = INFO or DEBUG, to control the verbosity of log
-# `app_log_dir` = the relative location of the application log file
-# `log_dir` = the relative location of the agent and command log file
-
- [server]
- hostname=localhost
- port=8440
- secured_port=8441
- check_path=/ws/v1/slider/agents/
- register_path=/ws/v1/slider/agents/{name}/register
- heartbeat_path=/ws/v1/slider/agents/{name}/heartbeat
-
- [agent]
- app_pkg_dir=app/definition
- app_install_dir=app/install
- app_run_dir=app/run
- app_task_dir=app/command-log
- app_log_dir=app/log
- app_tmp_dir=app/tmp
- log_dir=infra/log
- run_dir=infra/run
- version_file=infra/version
- log_level=INFO
-
- [python]
-
- [command]
- max_retries=2
- sleep_between_retries=1
-
- [security]
-
- [heartbeat]
- state_interval=6
- log_lines_count=300
-
-
-Once created, deploy the agent.ini file to HDFS:
-
- su yarn
-
- hdfs dfs -copyFromLocal agent.ini /slider/agent/conf
-
-## <a name="downsample"></a>Download Sample Application Packages
-
-There are three sample application packages available for download to use with Slider:
-
-<table>
- <tr>
- <td>Application</td>
- <td>Version</td>
- <td>URL</td>
- </tr>
- <tr>
- <td>Apache HBase</td>
- <td>0.96.0</td>
- <td>http://public-repo-1.hortonworks.com/slider/hbase_v096.tar</td>
- </tr>
- <tr>
- <td>Apache Storm</td>
- <td>0.9.1</td>
- <td>http://public-repo-1.hortonworks.com/slider/storm_v091.tar</td>
- </tr>
- <tr>
- <td>Apache Accumulo</td>
- <td>1.5.1</td>
- <td>http://public-repo-1.hortonworks.com/slider/accumulo_v151.tar</td>
- </tr>
-</table>
-
-
-Download the packages and deploy one of these sample applications to YARN via Slider using the steps below.
-
-## <a name="installapp"></a>Install, Configure, Start and Verify Sample Application
-
-* [Load Sample Application Package](#load)
-
-* [Create Application Specifications](#create)
-
-* [Start the Application](#start)
-
-* [Verify the Application](#verify)
-
-* [Manage the Application Lifecycle](#manage)
-
-### <a name="load"></a>Load Sample Application Package
-
- hdfs dfs -copyFromLocal *sample-application-package/slider
-
-If necessary, create HDFS folders needed by the application. For example, HBase requires the following HDFS-based setup:
-
- su hdfs
-
- hdfs dfs -mkdir /apps
-
- hdfs dfs -mkdir /apps/hbase
-
- hdfs dfs -chown yarn:hdfs /apps/hbase
-
-### <a name="create"></a>Create Application Specifications
-
-Configuring a Slider application consists of two parts: the [Resource Specification](#resspec),
- and the *[Application Configuration](#appconfig). Below are guidelines for creating these files.
-
-*Note: There are sample Resource Specifications (**resources.json**) and Application Configuration
-(**appConfig.json**) files in the *[Appendix](#appendixa)* and also in the root directory of the
-Sample Applications packages (e.g. /**hbase-v096/resources.json** and /**hbase-v096/appConfig.json**).*
-
-#### <a name="resspec"></a>Resource Specification
-
-Slider needs to know what components (and how many components) are in an application package to deploy. For example, in HBase, the components are **_master_** and **_worker_** -- the latter hosting **HBase RegionServers**, and the former hosting the **HBase Master**.
-
-As Slider creates each instance of a component in its own YARN container, it also needs to know what to ask YARN for in terms of **memory** and **CPU** for those containers.
-
-All this information goes into the **Resources Specification** file ("Resource Spec") named `resources.json`. The Resource Spec tells Slider how many instances of each component in the application (such as an HBase RegionServer) to deploy and the parameters for YARN.
-
-Sample Resource Spec files are available in the Appendix:
-
-* [Appendix A: Storm Sample Resource Specification](#heading=h.1hj8hn5xne7c)
-
-* [Appendix B: HBase Sample Resource Specification](#heading=h.l7z5mvhvxmzv)
-
-Store the Resource Spec file on your local disk (e.g. `/tmp/resources.json`).
-
-#### <a name="appconfig"></a>Application Configuration
-
-Alongside the Resource Spec there is the **Application Configuration** file ("App Config") which includes parameters that are specific to the application, rather than YARN. The App Config is a file that contains all application configuration. This configuration is applied to the default configuration provided by the application definition and then handed off to the associated component agent.
-
-For example, the heap sizes of the JVMs, The App Config defines the configuration details **specific to the application and component** instances. For HBase, this includes any values for the *to-be-generated *hbase-site.xml file, as well as options for individual components, such as their heap size.
-
-Sample App Configs are available in the Appendix:
-
-* [Appendix A: Storm Sample Application Configuration](#heading=h.2qai3c6w260l)
-
-* [Appendix B: HBase Sample Application Configuration](#heading=h.hetv1wn44c5x)
-
-Store the appConfig.json file on your local disc and a copy in HDFS:
-
- su yarn
-
- hdfs dfs -mkdir /slider/appconf
-
- hdfs dfs -copyFromLocal appConf.json /slider/appconf
-
-### <a name="start"></a>Start the Application
-
-Once the steps above are completed, the application can be started through the **Slider Command Line Interface (CLI)**.
-
-Change directory to the "bin" directory under the slider installation
-
- cd ${slider-install-dir}/slider-0.30.0/bin
-
-Execute the following command:
-
- ./slider create cl1 --manager yourResourceManagerHost:8050 --image hdfs://yourNameNodeHost:8020/slider/agent/slider-agent-0.30.0.tar.gz --template appConfig.json --resources resources.json
-
-### <a name="verify"></a>Verify the Application
-
-The successful launch of the application can be verified via the YARN Resource Manager Web UI. In most instances, this UI is accessible via a web browser at port 8088 of the Resource Manager Host:
-
-
-
-The specific information for the running application is accessible via the "ApplicationMaster" link that can be seen in the far right column of the row associated with the running application (probably the top row):
-
-
-
-### <a name="manage"></a>Manage the Application Lifecycle
-
-Once started, applications can be frozen/stopped, thawed/restarted, and destroyed/removed as follows:
-
-#### Frozen:
-
- ./slider freeze cl1 --manager yourResourceManagerHost:8050 --filesystem hdfs://yourNameNodeHost:8020
-
-#### Thawed:
-
- ./slider thaw cl1 --manager yourResourceManagerHost:8050 --filesystem hdfs://yourNameNodeHost:8020
-
-#### Destroyed:
-
- ./slider destroy cl1 --manager yourResourceManagerHost:8050 --filesystem hdfs://yourNameNodeHost:8020
-
-#### Flexed:
-
- ./slider flex cl1 --component worker 5 --manager yourResourceManagerHost:8050 --filesystem hdfs://yourNameNodeHost:8020
-
-# <a name="appendixa"></a>Appendix A: Apache Storm Sample Application Specifications
-
-## Storm Resource Specification Sample
-
- {
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- },
- "components" : {
- "slider-appmaster" : {
- },
- "NIMBUS" : {
- "yarn.role.priority" : "1",
- "yarn.component.instances" : "1"
- },
- "STORM_REST_API" : {
- "yarn.role.priority" : "2",
- "yarn.component.instances" : "1"
- },
- "STORM_UI_SERVER" : {
- "yarn.role.priority" : "3",
- "yarn.component.instances" : "1"
- },
- "DRPC_SERVER" : {
- "yarn.role.priority" : "4",
- "yarn.component.instances" : "1"
- },
- "SUPERVISOR" : {
- "yarn.role.priority" : "5",
- "yarn.component.instances" : "1"
- }
- }
- }
-
-
-## Storm Application Configuration Sample
-
- {
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- "A site property for type XYZ with name AA": "its value",
- "site.XYZ.AA": "Value",
- "site.hbase-site.hbase.regionserver.port": "0",
- "site.core-site.fs.defaultFS": "${NN_URI}",
- "Using a well known keyword": "Such as NN_HOST for name node host",
- "site.hdfs-site.dfs.namenode.http-address": "${NN_HOST}:50070",
- "a global property used by app scripts": "not affiliated with any site-xml",
- "site.global.app_user": "yarn",
- "Another example of available keywords": "Such as AGENT_LOG_ROOT",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- }
- }
-
-
-# <a name="appendixb"></a>Appendix B: Apache HBase Sample Application Specifications
-
-## HBase Resource Specification Sample
-
- {
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- },
- "components" : {
- "HBASE_MASTER" : {
- "yarn.role.priority" : "1",
- "yarn.component.instances" : "1"
- },
- "slider-appmaster" : {
- },
- "HBASE_REGIONSERVER" : {
- "yarn.role.priority" : "2",
- "yarn.component.instances" : "1"
- }
- }
- }
-
-
-## HBase Application Configuration Sample
-
- {
- "schema" : "http://example.org/specification/v2.0.0",
- "metadata" : {
- },
- "global" : {
- "agent.conf": "/slider/agent/conf/agent.ini",
- "agent.version": "/slider/agent/version",
- "application.def": "/slider/hbase_v096.tar",
- "config_types": "core-site,hdfs-site,hbase-site",
- "java_home": "/usr/jdk64/jdk1.7.0_45",
- "package_list": "files/hbase-0.96.1-hadoop2-bin.tar",
- "site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
- "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
- "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-0.96.1-hadoop2",
- "site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.global.hbase_master_heapsize": "1024m",
- "site.global.hbase_regionserver_heapsize": "1024m",
- "site.global.user_group": "hadoop",
- "site.global.security_enabled": "false",
- "site.hbase-site.hbase.hstore.flush.retries.number": "120",
- "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
- "site.hbase-site.hbase.hstore.compactionThreshold": "3",
- "site.hbase-site.hbase.rootdir": "${NN_URI}/apps/hbase/data",
- "site.hbase-site.hbase.stagingdir": "${NN_URI}/apps/hbase/staging",
- "site.hbase-site.hbase.regionserver.handler.count": "60",
- "site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38",
- "site.hbase-site.hbase.hregion.memstore.block.multiplier": "2",
- "site.hbase-site.hbase.hregion.memstore.flush.size": "134217728",
- "site.hbase-site.hbase.superuser": "yarn",
- "site.hbase-site.hbase.zookeeper.property.clientPort": "2181",
- "site.hbase-site.hbase.regionserver.global.memstore.upperLimit": "0.4",
- "site.hbase-site.zookeeper.session.timeout": "30000",
- "site.hbase-site.hbase.tmp.dir": "${AGENT_WORK_ROOT}/work/app/tmp",
- "site.hbase-site.hbase.local.dir": "${hbase.tmp.dir}/local",
- "site.hbase-site.hbase.hregion.max.filesize": "10737418240",
- "site.hbase-site.hfile.block.cache.size": "0.40",
- "site.hbase-site.hbase.security.authentication": "simple",
- "site.hbase-site.hbase.defaults.for.version.skip": "true",
- "site.hbase-site.hbase.zookeeper.quorum": "${ZK_HOST}",
- "site.hbase-site.zookeeper.znode.parent": "/hbase-unsecure",
- "site.hbase-site.hbase.hstore.blockingStoreFiles": "10",
- "site.hbase-site.hbase.hregion.majorcompaction": "86400000",
- "site.hbase-site.hbase.security.authorization": "false",
- "site.hbase-site.hbase.cluster.distributed": "true",
- "site.hbase-site.hbase.hregion.memstore.mslab.enabled": "true",
- "site.hbase-site.hbase.client.scanner.caching": "100",
- "site.hbase-site.hbase.zookeeper.useMulti": "true",
- "site.hbase-site.hbase.regionserver.info.port": "0",
- "site.hbase-site.hbase.master.info.port": "60010",
- "site.hbase-site.hbase.regionserver.port": "0",
- "site.core-site.fs.defaultFS": "${NN_URI}",
- "site.hdfs-site.dfs.namenode.https-address": "${NN_HOST}:50470",
- "site.hdfs-site.dfs.namenode.http-address": "${NN_HOST}:50070"
- }
- }
-
-
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/209cee43/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/index.md b/src/site/markdown/index.md
deleted file mode 100644
index 2c53ec7..0000000
--- a/src/site/markdown/index.md
+++ /dev/null
@@ -1,94 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-
-# Apache Slider: Dynamic YARN Applications
-
-
-
-Apache Slider is a YARN application to deploy existing distributed applications on YARN,
-monitor them and make them larger or smaller as desired -even while
-the application is running.
-
-Applications can be stopped, "frozen" and restarted, "thawed" later; the distribution
-of the deployed application across the YARN cluster is persisted -enabling
-a best-effort placement close to the previous locations on a cluster thaw.
-Applications which remember the previous placement of data (such as HBase)
-can exhibit fast start-up times from this feature.
-
-YARN itself monitors the health of "YARN containers" hosting parts of
-the deployed application -it notifies the Slider manager application of container
-failure. Slider then asks YARN for a new container, into which Slider deploys
-a replacement for the failed component. As a result, Slider can keep the
-size of managed applications consistent with the specified configuration, even
-in the face of failures of servers in the cluster -as well as parts of the
-application itself
-
-Some of the features are:
-
-* Allows users to create on-demand applications in a YARN cluster
-
-* Allow different users/applications to run different versions of the application.
-
-* Allow users to configure different application instances differently
-
-* Stop / Suspend / Resume application instances as needed
-
-* Expand / shrink application instances as needed
-
-The Slider tool is a Java command line application.
-
-The tool persists the information as JSON documents in HDFS.
-
-Once the cluster has been started, the cluster can be made to grow or shrink
-using the Slider commands. The cluster can also be stopped, *frozen*
-and later resumed, *thawed*.
-
-Slider implements all its functionality through YARN APIs and the existing
-application shell scripts. The goal of the application was to have minimal
-code changes and as of this writing, it has required few changes.
-
-## Using
-
-* [Getting Started](getting_started.html)
-* [Man Page](manpage.html)
-* [Examples](examples.html)
-* [Client Configuration](client-configuration.html)
-* [Client Exit Codes](exitcodes.html)
-* [Security](security.html)
-* [How to define a new slider-packaged application](slider_specs/index.html)
-* [Application configuration model](configuration/index.html)
-
-
-## Developing
-
-* [Architecture](architecture/index.html)
-* [Developing](developing/index.html)
-* [Application Needs](slider_specs/application_needs.md)
-* [Service Registry](registry/index.html)
-
-## Disclaimer
-
-Apache Slider (incubating) is an effort undergoing incubation at The
-Apache Software Foundation (ASF), sponsored by the name of Apache TLP
-sponsor. Incubation is required of all newly accepted projects until a
-further review indicates that the infrastructure, communications, and
-decision making process have stabilized in a manner consistent with
-other successful ASF projects. While incubation status is not
-necessarily a reflection of the completeness or stability of the code,
-it does indicate that the project has yet to be fully endorsed by the
-ASF.
[04/50] [abbrv] git commit: Merge branch
'feature/SLIDER_126_handle_heartbeat_failures' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER_126_handle_heartbeat_failures' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/45f5d796
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/45f5d796
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/45f5d796
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: 45f5d7968d03f24c9d6f1af03f9afb4c1925e8c7
Parents: 1a8062e 3aca57d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Jun 21 18:22:00 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Jun 21 18:22:00 2014 -0700
----------------------------------------------------------------------
slider-agent/conf/agent.ini | 2 +
.../src/main/python/agent/AgentConfig.py | 21 +++
slider-agent/src/main/python/agent/Constants.py | 3 +
.../src/main/python/agent/Controller.py | 23 ++-
slider-agent/src/main/python/agent/main.py | 4 +
.../src/test/python/agent/TestController.py | 30 ++++
slider-agent/src/test/python/agent/TestMain.py | 5 +-
.../providers/AbstractProviderService.java | 15 +-
.../slider/providers/ProviderService.java | 5 +-
.../slider/providers/agent/AgentKeys.java | 3 +
.../providers/agent/AgentLaunchParameter.java | 130 ++++++++++++++
.../providers/agent/AgentProviderService.java | 179 ++++++++++++++++---
.../slider/providers/agent/AgentRoles.java | 18 +-
.../apache/slider/providers/agent/Command.java | 13 +-
.../slider/providers/agent/CommandResult.java | 16 +-
.../providers/agent/ComponentInstanceState.java | 24 +++
.../slider/providers/agent/ContainerState.java | 41 +++++
.../providers/agent/HeartbeatMonitor.java | 116 ++++++++++++
.../server/appmaster/AMViewForProviders.java | 27 +++
.../server/appmaster/SliderAppMaster.java | 32 +++-
.../slider/server/appmaster/state/AppState.java | 20 +++
.../test_command_log/appConfig_fast_no_reg.json | 29 +++
.../test_command_log/appConfig_no_hb.json | 29 +++
.../model/mock/MockProviderService.groovy | 4 +-
.../agent/TestAgentLaunchParameter.java | 76 ++++++++
.../providers/agent/TestHeartbeatMonitor.java | 136 ++++++++++++++
.../publisher/TestAgentProviderService.java | 5 +-
.../lifecycle/AgentCommandTestBase.groovy | 100 ++++++++---
.../lifecycle/TestAgentClusterLifecycle.groovy | 2 +-
.../funtest/lifecycle/TestAgentFailures.groovy | 103 +++++++++++
.../funtest/lifecycle/TestAgentFailures2.groovy | 103 +++++++++++
.../lifecycle/TestAppsThroughAgent.groovy | 90 ++++------
32 files changed, 1253 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/45f5d796/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------