You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2014/06/24 21:46:39 UTC
svn commit: r1605167 [5/9] - in /incubator/slider/site: branches/ trunk/
trunk/cgi-bin/ trunk/content/ trunk/content/css/ trunk/content/design/
trunk/content/design/registry/ trunk/content/design/specification/
trunk/content/developing/ trunk/content/d...
Added: incubator/slider/site/trunk/content/developing/building.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/building.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/developing/building.md (added)
+++ incubator/slider/site/trunk/content/developing/building.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,374 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Building Apache Slider
+
+
+Here's how to set this up.
+
+## Before you begin
+
+### Networking
+
+The network on the development system must be functional, with hostname lookup
+of the local host working. Tests will fail without this.
+
+### Java 7
+
+Slider is built on Java 7 -please have a JDK for Java 7 or 8 set up
+
+### Maven
+
+You will need a version of Maven 3.0+, set up with enough memory
+
+ MAVEN_OPTS=-Xms256m -Xmx512m -Djava.awt.headless=true
+
+
+*Important*: As of October 6, 2013, Maven 3.1 is not supported due to
+[version issues](https://cwiki.apache.org/confluence/display/MAVEN/AetherClassNotFound).
+
+### Protoc
+
+You need a copy of the `protoc` compiler for protobuf compilation
+
+1. OS/X: `brew install protobuf`
+1. Others: consult (Building Hadoop documentation)[http://wiki.apache.org/hadoop/HowToContribute].
+
+The version of `protoc` installed must be the same as that used by Hadoop itself.
+This is absolutely critical to prevent JAR version problems.
+
+## Building a compatible Hadoop version
+
+
+Slider is built against Hadoop 2 -you can download and install
+a copy from the [Apache Hadoop Web Site](http://hadoop.apache.org).
+
+
+During development, its convenient (but not mandatory)
+to have a local version of Hadoop -so that we can find and fix bugs/add features in
+Hadoop as well in Slider.
+
+
+To build and install locally, check out apache svn/github, branch `release-2.4.0`,
+and create a branch off that tag
+
+ git clone git://git.apache.org/hadoop-common.git
+ cd hadoop-common
+ git remote rename origin apache
+ git fetch --tags apache
+ git checkout release-2.4.0 --
+ git checkout -b release-2.4.0
+
+
+For the scripts below, set the `HADOOP_VERSION` variable to the version
+
+ export HADOOP_VERSION=2.4.0
+
+or, for building against a pre-release version of Hadoop 2.4
+
+ git checkout branch-2
+ export HADOOP_VERSION=2.4.0-SNAPSHOT
+
+To build and install it locally, skipping the tests:
+
+ mvn clean install -DskipTests
+
+To make a tarball for use in test runs:
+
+ #On osx
+ mvn clean install package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip=true
+
+ # on linux
+ mvn clean package -Pdist -Pnative -Dtar -DskipTests -Dmaven.javadoc.skip=true
+
+Then expand this
+
+ pushd hadoop-dist/target/
+ gunzip hadoop-$HADOOP_VERSION.tar.gz
+ tar -xvf hadoop-$HADOOP_VERSION.tar
+ popd
+
+This creates an expanded version of Hadoop. You can now actually run Hadoop
+from this directory. Do note that unless you have the native code built for
+your target platform, Hadoop will be slower.
+
+## building a compatible HBase version
+
+If you need to build a version of HBase -rather than use a released version,
+here are the instructions (for the hbase-0.98 release branch)
+
+Checkout the HBase `trunk` branch from apache svn/github.
+
+
+ git clone git://git.apache.org/hbase.git
+ cd hbase
+ git remote rename origin apache
+ git fetch --tags apache
+
+then
+
+ git checkout -b apache/0.98
+or
+
+ git checkout tags/0.98.1
+
+If you have already been building versions of HBase, remove the existing
+set of artifacts for safety:
+
+ rm -rf ~/.m2/repository/org/apache/hbase/
+
+The maven command for building hbase artifacts against this hadoop version is
+
+ mvn clean install assembly:single -DskipTests -Dmaven.javadoc.skip=true
+
+To use a different version of Hadoop from that defined in the `hadoop-two.version`
+property of`/pom.xml`:
+
+ mvn clean install assembly:single -DskipTests -Dmaven.javadoc.skip=true -Dhadoop-two.version=$HADOOP_VERSION
+
+This will create an hbase `tar.gz` file in the directory `hbase-assembly/target/`
+in the hbase source tree.
+
+ export HBASE_VERSION=0.98.1
+
+ pushd hbase-assembly/target
+ gunzip hbase-$HBASE_VERSION-bin.tar.gz
+ tar -xvf hbase-$HBASE_VERSION-bin.tar
+ gzip hbase-$HBASE_VERSION-bin.tar
+ popd
+
+This will create an untarred directory containing
+hbase. Both the `.tar.gz` and untarred file are needed for testing. Most
+tests just work directly with the untarred file as it saves time uploading
+and downloading then expanding the file.
+
+(and if you set `HBASE_VERSION` to something else, you can pick up that version
+-making sure that slider is in sync)
+
+For more information (including recommended Maven memory configuration options),
+see [HBase building](http://hbase.apache.org/book/build.html)
+
+For building just the JAR files:
+
+ mvn clean install -DskipTests -Dhadoop.profile=2.0 -Dhadoop-two.version=$HADOOP_VERSION
+
+*Tip:* you can force set a version in Maven by having it update all the POMs:
+
+ mvn versions:set -DnewVersion=0.98.1-SNAPSHOT
+
+## Building Accumulo
+
+Clone accumulo from apache;
+
+ git clone http://git-wip-us.apache.org/repos/asf/accumulo.git
+
+
+Check out branch 1.6.1-SNAPSHOT
+
+ git checkout 1.6.1-SNAPSHOT
+
+In the accumulo project directory, build it
+
+ mvn clean install -Passemble -DskipTests -Dmaven.javadoc.skip=true \
+ -Dhadoop.profile=2
+
+The default Hadoop version for accumulo-1.6.1 is hadoop 2.4.0; to build
+against a different version use the command
+
+ mvn clean install -Passemble -DskipTests -Dmaven.javadoc.skip=true \
+ -Dhadoop.profile=2 -Dhadoop.version=$HADOOP_VERSION
+
+This creates an accumulo tar.gz file in `assemble/target/`. Extract this
+to create an expanded directory
+
+ accumulo/assemble/target/accumulo-1.6.1-SNAPSHOT-bin.tar.gz
+
+ This can be done with the command sequence
+
+ export ACCUMULO_VERSION=1.6.1-SNAPSHOT
+
+ pushd assemble/target/
+ gunzip -f accumulo-$ACCUMULO_VERSION-bin.tar.gz
+ tar -xvf accumulo-$ACCUMULO_VERSION-bin.tar.gz
+ popd
+
+Note that the final location of the accumulo files is needed for the configuration,
+it may be directly under target/ or it may be in a subdirectory, with
+a path such as `target/accumulo-$ACCUMULO_VERSION-dev/accumulo-$ACCUMULO_VERSION/`
+
+
+## Testing
+
+### Configuring Slider to locate the relevant artifacts
+
+You must have the file `src/test/resources/slider-test.xml` (this
+is ignored by git), declaring where HBase, accumulo, Hadoop and zookeeper are:
+
+ <configuration>
+
+ <property>
+ <name>slider.test.hbase.home</name>
+ <value>/home/slider/hbase/hbase-assembly/target/hbase-0.98.0-SNAPSHOT</value>
+ <description>HBASE Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <value>/home/slider/hbase/hbase-assembly/target/hbase-0.98.0-SNAPSHOT-bin.tar.gz</value>
+ <description>HBASE archive URI</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.home</name>
+ <value>/home/slider/accumulo/assemble/target/accumulo-1.6.1-SNAPSHOT/</value>
+ <description>Accumulo Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <value>/home/slider/accumulo/assemble/target/accumulo-1.6.1-SNAPSHOT-bin.tar.gz</value>
+ <description>Accumulo archive URI</description>
+ </property>
+
+ <property>
+ <name>zk.home</name>
+ <value>
+ /home/slider/Apps/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>
+ /home/slider/hadoop-common/hadoop-dist/target/hadoop-2.3.0</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+ </configuration>
+
+
+## Debugging a failing test
+
+1. Locate the directory `target/$TESTNAME` where TESTNAME is the name of the
+test case and or test method. This directory contains the Mini YARN Cluster
+logs. For example, `TestLiveRegionService` stores its data under
+`target/TestLiveRegionService`
+
+1. Look under that directory for `-logdir` directories, then an application
+and container containing logs. There may be more than node being simulated;
+every node manager creates its own logdir.
+
+1. Look for the `out.txt` and `err.txt` files for stdout and stderr log output.
+
+1. Slider uses SLF4J to log to `out.txt`; remotely executed processes may use
+either stream for logging
+
+Example:
+
+ target/TestLiveRegionService/TestLiveRegionService-logDir-nm-1_0/application_1376095770244_0001/container_1376095770244_0001_01_000001/out.txt
+
+1. The actual test log from JUnit itself goes to the console and into
+`target/surefire/`; this shows the events happening in the YARN services as well
+ as (if configured) HDFS and Zookeeper. It is noisy -everything after the *teardown*
+ message happens during cluster teardown, after the test itself has been completed.
+ Exceptions and messages here can generally be ignored.
+
+This is all a bit complicated -debugging is simpler if a single test is run at a
+time, which is straightforward
+
+ mvn clean test -Dtest=TestLiveRegionService
+
+
+### Building the JAR file
+
+You can create the JAR file and set up its directories with
+
+ mvn package -DskipTests
+
+# Development Notes
+
+<!---
+## Git branch model
+
+
+The git branch model uses is
+[Git Flow](http://nvie.com/posts/a-successful-git-branching-model/).
+
+This is a common workflow model for Git, and built in to
+[Atlassian Source Tree](http://sourcetreeapp.com/).
+
+The command line `git-flow` tool is easy to install
+
+ brew install git-flow
+
+or
+
+ apt-get install git-flow
+
+You should then work on all significant features in their own branch and
+merge them back in when they are ready.
+
+
+ # until we get a public JIRA we're just using an in-house one. sorry
+ git flow feature start BUG-8192
+
+ # finishes merges back in to develop/
+ git flow feature finish BUG-8192
+
+ # release branch
+ git flow release start 0.4.0
+
+ git flow release finish 0.4.0
+-->
+
+## Attention OS/X developers
+
+YARN on OS/X doesn't terminate subprocesses the way it does on Linux, so
+HBase Region Servers created by the hbase shell script remain running
+even after the tests terminate.
+
+This causes some tests -especially those related to flexing down- to fail,
+and test reruns may be very confused. If ever a test fails because there
+are too many region servers running, this is the likely cause
+
+After every test run: do a `jps -v` to look for any leftover HBase services
+-and kill them.
+
+Here is a handy bash command to do this
+
+ jps -l | grep HRegion | awk '{print $1}' | xargs kill -9
+
+
+## Groovy
+
+Slider uses Groovy 2.x as its language for writing tests -for better assertions
+and easier handling of lists and closures. Although the first prototype
+used Groovy on the production source, this was dropped in favor of
+a Java-only production codebase.
+
+## Maven utils
+
+
+Here are some handy aliases to make maven easier
+
+ alias mci='mvn clean install -DskipTests'
+ alias mi='mvn install -DskipTests'
+ alias mvct='mvn clean test'
+ alias mvnsite='mvn site:site -Dmaven.javadoc.skip=true'
+ alias mvt='mvn test'
+
+
Propchange: incubator/slider/site/trunk/content/developing/building.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/developing/functional_tests.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/functional_tests.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/developing/functional_tests.md (added)
+++ incubator/slider/site/trunk/content/developing/functional_tests.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,416 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Testing Apache Slider
+
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
+ NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
+ "OPTIONAL" in this document are to be interpreted as described in
+ RFC 2119.
+
+# Functional Tests
+
+The functional test suite is designed to test slider against
+a live cluster.
+
+For these to work you need
+
+1. A YARN Cluster -secure or insecure
+1. A `slider-client.xml` file configured to interact with the cluster
+1. Agent
+1. HBase tests: HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+directory
+1. Accumulo Tests Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+directory
+
+## Configuration of functional tests
+
+Maven needs to be given
+
+1. A path to the expanded test archive
+1. A path to a slider configuration directory for the cluster
+
+The path for the expanded test is automatically calculated as being the directory under
+`..\slider-assembly\target` where an untarred slider distribution can be found.
+If it is not present, the tests will fail
+
+The path to the configuration directory must be supplied in the property
+`slider.conf.dir` which can be set on the command line
+
+ mvn test -Dslider.conf.dir=src/test/configs/sandbox/slider
+
+It can also be set in the (optional) file `slider-funtest/build.properties`:
+
+ slider.conf.dir=src/test/configs/sandbox/slider
+
+This file is loaded whenever a slider build or test run takes place
+
+## Configuration of `slider-client.xml`
+
+The `slider-client.xml` must have extra configuration options for both the HBase and
+Accumulo tests, as well as a common set for actually talking to a YARN cluster.
+
+## Disabling the functional tests entirely
+
+All functional tests which require a live YARN cluster
+can be disabled through the property `slider.funtest.enabled`
+
+ <property>
+ <name>slider.funtest.enabled</name>
+ <value>false</value>
+ </property>
+
+There is a configuration do do exactly this in
+`src/test/configs/offline/slider`:
+
+ slider.conf.dir=src/test/configs/offline/slider
+
+Tests which do not require a live YARN cluster will still run;
+these verify that the `bin/slider` script works.
+
+### Non-mandatory options
+
+The following test options may be added to `slider-client.xml` if the defaults
+need to be changed
+
+ <property>
+ <name>slider.test.zkhosts</name>
+ <description>comma separated list of ZK hosts</description>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <name>slider.test.thaw.wait.seconds</name>
+ <description>Time to wait in seconds for a thaw to result in a running AM</description>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>slider.test.freeze.wait.seconds</name>
+ <description>Time to wait in seconds for a freeze to halt the cluster</description>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <name>slider.test.timeout.millisec</name>
+ <description>Time out in milliseconds before a test is considered to have failed.
+ There are some maven properties which also define limits and may need adjusting</description>
+ <value>180000</value>
+ </property>
+
+ <property>
+ <name>slider.test.yarn.ram</name>
+ <description>Size in MB to ask for containers</description>
+ <value>192</value>
+ </property>
+
+
+Note that while the same properties need to be set in
+`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
+filesystem -here a URI to a path visible across all nodes in the cluster are required
+the tests do not copy the .tar/.tar.gz files over. The application configuration
+directories may be local or remote -they are copied into the `.slider` directory
+during cluster creation.
+
+## Provider-specific parameters
+
+An individual provider can pick up settings from their own
+`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
+We strongly advice placing all the values in the `slider-core` file.
+
+1. All uncertainty about which file is picked up on the class path first goes
+away
+2. There's one place to keep all the configuration values in sync.
+
+### Agent Tests
+
+Agent tests are executed through the following mvn command executed at slider/slider-funtest:
+
+```
+cd slider-funtest
+mvn test -Dslider.conf.dir=../src/test/clusters/remote/slider -Dtest=TestAppsThroughAgent -DfailIfNoTests=false
+```
+
+**Enable/Execute the tests**
+
+To enable the test ensure that *slider.test.agent.enabled* is set to *true*.
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+**Test setup**
+
+Edit config file src/test/clusters/remote/slider/slider-client.xml and ensure that the host names are accurate for the test cluster.
+
+**User setup**
+
+Ensure that the user, running the test, is present on the cluster against which you are running the tests. The user must be a member of the hadoop group.
+
+E.g. adduser **testuser** -d /home/**testuser** -G hadoop -m
+
+**HDFS Setup**
+
+Set up hdfs folders for slider and test user
+
+* su hdfs
+* hdfs dfs -mkdir /slider
+* hdfs dfs -chown testuser:hdfs /slider
+* hdfs dfs -mkdir /user/testuser
+* hdfs dfs -chown testuser:hdfs /user/testuser
+
+Load up agent package and config
+
+* su **testuser**
+* hdfs dfs -mkdir /slider/agent
+* hdfs dfs -mkdir /slider/agent/conf
+* hdfs dfs -copyFromLocal SLIDER_INSTALL_LOC/agent/conf/agent.ini /slider/agent/conf
+
+Ensure correct host name is provided for the agent tarball.
+
+ <property>
+ <name>slider.test.agent.tar</name>
+ <description>Path to the Agent Tar file in HDFS</description>
+ <value>hdfs://NN_HOSTNAME:8020/slider/agent/slider-agent.tar.gz</value>
+ </property>
+
+
+
+### HBase Tests
+
+The HBase tests can be enabled or disabled
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+Mandatory test parameters must be added to `slider-client.xml`
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value>hdfs://sandbox:8020/user/slider/hbase.tar.gz</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value>file://${user.dir}/src/test/configs/sandbox/hbase</value>
+ </property>
+
+Optional parameters:
+
+ <property>
+ <name>slider.test.hbase.launch.wait.seconds</name>
+ <description>Time to wait in seconds for HBase to start</description>
+ <value>1800</value>
+ </property>
+
+#### Accumulo configuration options
+
+Enable/disable the tests
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+Optional parameters
+
+ <property>
+ <name>slider.test.accumulo.launch.wait.seconds</name>
+ <description>Time to wait in seconds for Accumulo to start</description>
+ <value>1800</value>
+ </property>
+
+### Configuring the YARN cluster for tests
+
+Here are the configuration options we use in `yarn-site.xml` for testing:
+
+These tell YARN to ignore memory requirements in allocating VMs, and
+to keep the log files around after an application run.
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>1</value>
+ </property>
+ <property>
+ <description>Whether physical memory limits will be enforced for
+ containers.
+ </description>
+ <name>yarn.nodemanager.pmem-check-enabled</name>
+ <value>false</value>
+ </property>
+ <!-- we really don't want checking here-->
+ <property>
+ <name>yarn.nodemanager.vmem-check-enabled</name>
+ <value>false</value>
+ </property>
+
+ <!-- how long after a failure to see what is left in the directory-->
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>60000</value>
+ </property>
+
+ <!--ten seconds before the process gets a -9 -->
+ <property>
+ <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
+ <value>30000</value>
+ </property>
+
+
+## Testing against a secure cluster
+
+To test against a secure cluster
+
+1. `slider-client.xml` must be configured as per [Security](/docs/security.html).
+1. the client must have the kerberos tokens issued so that the user running
+the tests has access to HDFS and YARN.
+
+If there are problems authenticating (including the cluster being offline)
+the tests appear to hang
+
+### Validating the configuration
+
+ mvn test -Dtest=TestBuildSetup
+
+### Using relative paths in test configurations
+
+When you are sharing configurations across machines via SCM or similar,
+its impossible to have absolute paths in the configuration options to
+the location of items in the local filesystem (e.g. configuration directories).
+
+There's two techniques
+
+1. Keep the data in HDFS and refer to it there. This works if there is a shared,
+persistent HDFS cluster.
+
+1. Use the special property `slider.test.conf.dir` that is set to the path
+of the directory, and which can then be used to create an absolute path
+from paths relative to the configuration dir:
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value>file://${slider.test.conf.dir}/../hbase</value>
+ </property>
+
+
+If the actual XML file path is required, a similar property
+`slider.test.conf.xml` is set.
+
+
+## Parallel execution
+
+Attempts to run test cases in parallel failed -even with a configuration
+to run methods in a class sequentially, but separate classes independently.
+
+Even after identifying and eliminating some unintended sharing of static
+mutable variables, trying to run test cases in parallel seemed to hang
+tests and produce timeouts.
+
+For this reason parallel tests have been disabled. To accelerate test runs
+through parallelization, run different tests on different hosts instead.
+
+## Other constraints
+
+* Port assignments SHOULD NOT be fixed, as this will cause clusters to fail if
+there are too many instances of a role on a same host, or if other tests are
+using the same port.
+* If a test does need to fix a port, it MUST be for a single instance of a role,
+and it must be different from all others. The assignment should be set in
+`org.apache.slider.funtest.itest.PortAssignments` so as to ensure uniqueness
+over time. Otherwise: use the value of `0` to allow the OS to assign free ports
+on demand.
+
+## Test Requirements
+
+
+1. Test cases should be written so that each class works with exactly one
+Slider-deployed cluster
+1. Every test MUST have its own cluster name -preferably derived from the
+classname.
+1. This cluster should be deployed in an `@BeforeClass` method.
+1. The `@AfterClass` method MUST tear this cluster down.
+1. Tests must skip their execution if functional tests -or the
+specific hbase or accumulo categories- are disabled.
+1. Tests within the suite (i.e. class) must be designed to be independent
+-to work irrespectively of the ordering of other tests.
+
+## Running and debugging the functional tests.
+
+The functional tests all
+
+1. In the root `slider` directory, build a complete Slider release
+
+ mvn install -DskipTests
+1. Start the YARN cluster/set up proxies to connect to it, etc.
+
+1. In the `slider-funtest` dir, run the tests
+
+ mvn test
+
+A common mistake during development is to rebuild the `slider-core` JARs
+then the `slider-funtest` tests without rebuilding the `slider-assembly`.
+In this situation, the tests are in sync with the latest build of the code
+-including any bug fixes- but the scripts executed by those tests are
+of a previous build of `slider-core.jar`. As a result, the fixes are not picked
+up.
+
+#### To propagate changes in slider-core through to the funtest classes for
+testing, you must build/install all the slider packages from the root assembly.
+
+ mvn clean install -DskipTests
+
+## Limitations of slider-funtest
+
+1. All tests run from a single client -workload can't scale
+1. Output from failed AM and containers aren't collected
+
+## Troubleshooting the functional tests
+
+1. If application instances fail to come up as there are still outstanding
+requests, it means that YARN didn't have the RAM/cores to spare for the number
+of containers. Edit the `slider.test.yarn.ram` to make it smaller.
+
+1. If you are testing in a local VM and stops responding, it'll have been
+swapped out to RAM. Rebooting can help, but for a long term fix go through
+all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
+smaller numbers, like 256M each. Also: turn off unused services (hcat, oozie,
+webHDFS)
+
+1. The YARN UI will list the cluster launches -look for the one
+with a name close to the test and view its logs
+
+1. Container logs will appear "elsewhere". The log lists
+the containers used -you may be able to track the logs
+down from the specific nodes.
+
+1. If you browse the filesystem, look for the specific test clusters
+in `~/.slider/cluster/$testname`
+
+1. If you are using a secure cluster, make sure that the clocks
+are synchronized, and that you have a current token -`klist` will
+tell you this. In a VM: install and enable `ntp`, consider rebooting if ther
+are any problems. Check also that it has the same time zone settings
+as the host OS.
Propchange: incubator/slider/site/trunk/content/developing/functional_tests.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/developing/index.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/index.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/developing/index.md (added)
+++ incubator/slider/site/trunk/content/developing/index.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,34 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Developing Apache Slider
+
+Slider is an open source project -anyone is free to contributed, and we
+strongly encourage people to do so.
+
+Here are documents covering how to go about building, testing and releasing
+Slider
+
+* [Building](/developing/building.html)
+* [Debugging](/docs/debugging.html)
+* [Testing](/developing/testing.html)
+* [Functional Testing](/developing/functional_tests.html)
+* [Manual Testing](/developing/manual_testing.html)
+* [Releasing](/developing/releasing.html)
+
+
+
Propchange: incubator/slider/site/trunk/content/developing/index.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/developing/manual_testing.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/manual_testing.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/developing/manual_testing.md (added)
+++ incubator/slider/site/trunk/content/developing/manual_testing.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,53 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Manually Testing Apache Slider
+
+Manual testing invloves using Slider package and an AppPackage to perform basic
+ cluster functionalities such as create/destroy, flex up/down, and freeze/thaw.
+ A python helper script is provided that can be used to automatically test and app package.
+
+## `SliderTester.py`
+Details to be added.
+
+## `SliderTester.ini`
+The various config parameters are:
+
+### slider
+* `package`: location of the slider package
+* `jdk.path`: jdk path on the test hosts
+
+### app
+* `package`: location of the app package
+
+### cluster
+* `yarn.application.classpath`: yarn application classpaths
+* `slider.zookeeper.quorum`: the ZK quorum hosts
+* `yarn.resourcemanager.address`:
+* `yarn.resourcemanager.scheduler.address`:
+* `fs.defaultFS`: e.g. `hdfs://NN_HOST:8020`
+
+### test
+* `app.user`: user to use for app creation
+* `hdfs.root.user`: hdfs root user
+* `hdfs.root.dir`: HDFS root, default /slidertst
+* `hdfs.user.dir`: HDFS user dir, default /user
+* `test.root`: local test root folder, default /test
+* `cluster.name`: name of the test cluster, default tst1
+* `cluster.type`: cluster type to build and test, e.g. hbase,storm,accumulo
+
+### agent
Propchange: incubator/slider/site/trunk/content/developing/manual_testing.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/developing/releasing.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/releasing.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/developing/releasing.md (added)
+++ incubator/slider/site/trunk/content/developing/releasing.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,195 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+# Apache Slider Release Process
+
+Here is our release process.
+
+
+## IMPORTANT: THIS IS OUT OF DATE WITH THE MOVE TO THE ASF ##
+
+### Before you begin
+
+Check out the latest version of the develop branch,
+run the tests. This should be done on a checked out
+version of the code that is not the one you are developing on
+(ideally, a clean VM), to ensure that you aren't releasing a slightly
+modified version of your own, and that you haven't accidentally
+included passwords or other test run details into the build resource
+tree.
+
+The `slider-funtest` functional test package is used to run functional
+tests against a running Hadoop YARN cluster. It needs to be configured
+according to the instructions in [testing](/developing/testing.html) to
+create HBase and Accumulo clusters in the YARN cluster.
+
+*Make sure that the functional tests are passing (and not being skipped) before
+starting to make a release*
+
+
+
+**Step #1:** Create a JIRA for the release, estimate 3h
+(so you don't try to skip the tests)
+
+ export SLIDER_RELEASE_JIRA=SLIDER-13927
+
+**Step #2:** Check everything in. Git flow won't let you progress without this.
+
+**Step #3:** Git flow: create a release branch
+
+ export SLIDER_RELEASE=0.5.2
+
+ git flow release start slider-$SLIDER_RELEASE
+
+**Step #4:** in the new branch, increment those version numbers using (the maven
+versions plugin)[http://mojo.codehaus.org/versions-maven-plugin/]
+
+ mvn versions:set -DnewVersion=$SLIDER_RELEASE
+
+
+**Step #5:** commit the changed POM files
+
+ git add <changed files>
+ git commit -m "$SLIDER_RELEASE_JIRA updating release POMs for $SLIDER_RELEASE"
+
+
+**Step #6:** Do a final test run to make sure nothing is broken
+
+In the `slider` directory, run:
+
+ mvn clean install -DskipTests
+
+Once everything is built- including .tar files, run the tests
+
+ mvn test
+
+This will run the functional tests as well as the `slider-core` tests.
+
+It is wise to reset any VMs here, and on live clusters kill all running jobs.
+This stops functional tests failing because the job doesn't get started before
+the tests time out.
+
+As the test run takes 30-60+ minutes, now is a good time to consider
+finalizing the release notes.
+
+
+**Step #7:** Build the release package
+
+Run
+
+ mvn clean site:site site:stage package -DskipTests
+
+
+
+**Step #8:** validate the tar file
+
+Look in `slider-assembly/target` to find the `.tar.gz` file, and the
+expanded version of it. Inspect that expanded version to make sure that
+everything looks good -and that the versions of all the dependent artifacts
+look good too: there must be no `-SNAPSHOT` dependencies.
+
+
+**Step #9:** Build the release notes
+
+Create a a one-line plain text release note for commits and tags
+And a multi-line markdown release note, which will be used for artifacts.
+
+
+ Release against hadoop 2.4.0, HBase-0.98.1 and Accumulo 1.5.1 artifacts.
+
+The multi-line release notes should go into `slider/src/site/markdown/release_notes`.
+
+
+These should be committed
+
+ git add --all
+ git commit -m "$SLIDER_RELEASE_JIRA updating release notes"
+
+**Step #10:** End the git flow
+
+Finish the git flow release, either in the SourceTree GUI or
+the command line:
+
+
+ git flow release finish slider-$SLIDER_RELEASE
+
+
+On the command line you have to enter the one-line release description
+prepared earlier.
+
+You will now be back on the `develop` branch.
+
+**Step #11:** update mvn versions
+
+Switch back to `develop` and update its version number past
+the release number
+
+
+ export SLIDER_RELEASE=0.6.0-SNAPSHOT
+ mvn versions:set -DnewVersion=$SLIDER_RELEASE
+ git commit -a -m "$SLIDER_RELEASE_JIRA updating development POMs to $SLIDER_RELEASE"
+
+**Step #12:** Push the release and develop branches to github
+
+ git push origin master develop
+
+(assuming that `origin` maps to `git@github.com:hortonworks/slider.git`;
+ you can check this with `git remote -v`
+
+
+The `git-flow` program automatically pushes up the `release/slider-X.Y` branch,
+before deleting it locally.
+
+If you are planning on any release work of more than a single test run,
+consider having your local release branch track the master.
+
+
+**Step #13:** ### Release on github small artifacts
+
+Browse to https://github.com/hortonworks/slider/releases/new
+
+Create a new release on the site by following the instructions
+
+Files under 5GB can be published directly. Otherwise, follow step 14
+
+**Step #14:** For releasing via an external CDN (e.g. Rackspace Cloud)
+
+Using the web GUI for your particular distribution network, upload the
+`.tar.gz` artifact
+
+After doing this, edit the release notes on github to point to the
+tar file's URL.
+
+Example:
+ [Download slider-0.10.1-all.tar.gz](http://dffeaef8882d088c28ff-185c1feb8a981dddd593a05bb55b67aa.r18.cf1.rackcdn.com/slider-0.10.1-all.tar.gz)
+
+**Step #15:** Announce the release
+
+**Step #16:** Finish the JIRA
+
+Log the time, close the issue. This should normally be the end of a
+sprint -so wrap that up too.
+
+**Step #17:** Get back to developing!
+
+Check out the develop branch and purge all release artifacts
+
+ git checkout develop
+ git pull origin
+ mvn clean
+
Propchange: incubator/slider/site/trunk/content/developing/releasing.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/developing/testing.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/developing/testing.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/developing/testing.md (added)
+++ incubator/slider/site/trunk/content/developing/testing.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,182 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Testing Apache Slider
+
+ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
+ NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and
+ "OPTIONAL" in this document are to be interpreted as described in
+ RFC 2119.
+
+## Standalone Tests
+
+Slider core contains a suite of tests that are designed to run on the local machine,
+using Hadoop's `MiniDFSCluster` and `MiniYARNCluster` classes to create small,
+one-node test clusters. All the YARN/HDFS code runs in the JUnit process; the
+AM and spawned processeses run independently.
+
+
+
+### For HBase Tests in `slider-providers/hbase`
+
+Requirements
+* A copy of `hbase.tar.gz` in the local filesystem
+* A an expanded `hbase.tar.gz` in the local filesystem
+
+
+### For Accumulo Tests in `slider-providers/accumulo`
+* A copy of `accumulo.tar.gz` in the local filesystem,
+* An expanded `accumulo.tar.gz` in the local filesystem,
+* an expanded Zookeeper installation
+
+All of these need to be defined in the file `slider-core/src/test/resources/slider-test.xml`
+
+Example:
+
+ <configuration>
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.home</name>
+ <value>/home/slider/hbase-0.98.0</value>
+ <description>HBASE Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <value>/home/slider/Projects/hbase-0.98.0-bin.tar.gz</value>
+ <description>HBASE archive URI</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.home</name>
+ <value>
+ /home/slider/accumulo-1.6.0-SNAPSHOT/</value>
+ <description>Accumulo Home</description>
+ </property>
+
+ <property>
+ <name>slider.test.accumulo.tar</name>
+ <value>/home/slider/accumulo-1.6.0-SNAPSHOT-bin.tar</value>
+ <description>Accumulo archive URI</description>
+ </property>
+
+ <property>
+ <name>slider.test.am.restart.time</name>
+ <description>Time in millis to await an AM restart</description>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>zk.home</name>
+ <value>/home/slider/zookeeper</value>
+ <description>Zookeeper home dir on target systems</description>
+ </property>
+
+ <property>
+ <name>hadoop.home</name>
+ <value>/home/slider/hadoop-2.2.0</value>
+ <description>Hadoop home dir on target systems</description>
+ </property>
+
+ </configuration>
+
+*Important:* For the local tests, a simple local filesystem path is used for
+all the values.
+
+For the functional tests, the accumulo and hbase tar properties will
+need to be set to a URL of a tar file that is accessible to all the
+nodes in the cluster -which usually means HDFS, and so an `hdfs://` URL
+
+
+## Provider-specific parameters
+
+An individual provider can pick up settings from their own
+`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
+We strongly advice placing all the values in the `slider-core` file.
+
+1. All uncertainty about which file is picked up on the class path first goes
+away
+2. There's one place to keep all the configuration values in sync.
+
+### Agent Tests
+
+
+### HBase Tests
+
+The HBase tests can be enabled or disabled
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
+
+Mandatory test parameters must be added to `slider-client.xml`
+
+
+ <property>
+ <name>slider.test.hbase.tar</name>
+ <description>Path to the HBase Tar file in HDFS</description>
+ <value>hdfs://sandbox:8020/user/slider/hbase.tar.gz</value>
+ </property>
+
+ <property>
+ <name>slider.test.hbase.appconf</name>
+ <description>Path to the directory containing the HBase application config</description>
+ <value>file://${user.dir}/src/test/configs/sandbox/hbase</value>
+ </property>
+
+Optional parameters:
+
+ <property>
+ <name>slider.test.hbase.launch.wait.seconds</name>
+ <description>Time to wait in seconds for HBase to start</description>
+ <value>180000</value>
+ </property>
+
+
+#### Accumulo configuration options
+
+Enable/disable the tests
+
+ <property>
+ <name>slider.test.accumulo.enabled</name>
+ <description>Flag to enable/disable Accumulo tests</description>
+ <value>true</value>
+ </property>
+
+
+Optional parameters
+
+ <property>
+ <name>slider.test.accumulo.launch.wait.seconds</name>
+ <description>Time to wait in seconds for Accumulo to start</description>
+ <value>180000</value>
+ </property>
+
Propchange: incubator/slider/site/trunk/content/developing/testing.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/docs/client-configuration.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/client-configuration.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/client-configuration.md (added)
+++ incubator/slider/site/trunk/content/docs/client-configuration.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,310 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Apache Slider Client Configuration
+
+This document covers how the client application is itself configured.
+
+## Summary
+
+The client application can be configured
+
+1. On the command line, which can set client options and JVM system properties.
+2. With Hadoop-style configuration options in the file `slider-client.xml`
+ in the configuration directory`conf/` dir
+2. Or, if the environment variable `SLIDER_CONF_DIR` is set, in the
+ file `$SLIDER_CONF_DIR/slider-client.xml`
+1. Logging is defined in the `log4j.properties` file in the same configuration
+directory.
+1. VM options can be defined in `SLIDER_JVM_OPTS`
+
+The options defined in a Slider cluster configuration are only used by the client
+when creating a cluster -not for the actual client itself.
+
+## Introduction
+
+The Slider client needs to be configured to talk to a Hadoop filesystem and a
+YARN resource manager ("the RM"). In a secure cluster it needs to be told the Kerberos
+identity, the *principal* of both the HDFS namenode and the YARN RM -and it may
+also need some JVM options set in order for Java's Kerberos module to
+correctly identify itself to these services.
+
+It cannot rely on local `HADOOP_PREFIX/conf/hadoop-site.xml` and
+`$YARN_PREFIX/conf/yarn-site.xml` files -because it is designed to
+work on client machines that may not have Hadoop and YARN installed.
+
+Instead all client-side (non-JVM) options can be predefined in the
+configuration file `slider-client.xml`.
+
+## Setting Slider JVM options
+
+Core JVM options can be set in the environment variable `SLIDER_JVM_OPTS`;
+if unset the `bin/slider` script will use the default values that were
+current when that version of Slider was released. These values may change
+across versions, and may in fact be.
+
+At the time of writing, the default values were:
+
+ "-Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Xmx256m -Dslider.confdir=${confdir}"
+
+To allow some Java system properties to be set without editing this
+environment variable, such system properties may be set on the Slider command
+line through the `-S` parameter. For example, the following two operations are
+equivalent in terms of setting the system property `java.security.krb5.realm`
+to the value `LOCAL`.
+
+ export SLIDER_JVM_OPTS="-Djava.security.krb5.realm=LOCAL"
+
+and
+
+ slider -S java.security.krb5.realm=LOCAL
+
+Note that the first declaration invalidates all default JVM options; if any of
+those were desired, they should be included in the new definition.
+
+Multiple system property declarations are allowed on the command line -including
+duplicate declarations. In such a case the order of assignment is undefined.
+
+For any system property that the user expects to have to issue on every command
+-including any kerberos-related properties, adding them to the JVM options
+environment variable guarantees that they are always set.
+
+## Setting Slider client options on the command line with the `-D` parameter
+
+The slider client is configured via Hadoop-style configuration options.
+To be precise, all standard Hadoop-common, hadoop-hdfs client and hadoop-yar
+client-side options control how Slider communicates with the Hadoop YARN cluster.
+
+There are extra options specific to Slider itself, options which
+are again set as Hadoop configuration parameters.
+
+All Hadoop and Slider options can be set on the command line using the `-D`
+parameter followed by the appropriate `key=value` argument
+
+
+For example, here is a definition of the default Hadoop filesystem:
+
+ -D fs.defaultFS=hdfs://namenode:9000
+
+Multiple definitions are of course allowed on the command line
+
+ -D fs.defaultFS=hdfs://namenode:9000 -D dfs.namenode.kerberos.principal=hdfs/namenode@LOCAL
+
+Slider-specific options can be made the same way
+
+ -D slider.kerberos.principal=
+
+If duplicate declarations are made the order of assignment is undefined.
+
+# Setting common options through specific command-line arguments
+
+Some Hadoop and Slider options are so common that they have specific
+shortcut commands to aid their use
+
+`-m`, `--manager` : sets the YARN resource manager. Equivalent to setting the
+`yarn.resourcemanager.address` option
+
+`--fs`, `--filesystem`: defines the filesystem. Equivalent to setting the
+`fs.defaultFS` option
+
+If these shortcuts are used and the options are also defined via `-D`
+declarations, the order of assignment is undefined.
+
+# Defining Hadoop and Slider Options in the `slider-client.xml` file.
+
+In the Slider installation, alongside the `bin/slider` script is
+a configuration directory `conf`. This contains the files:
+
+1. `log4j.properties`
+1. `slider-client.xml`
+
+The `log4j.properties` file is not covered here -it is a standard Log4J file.
+At the time of writing, this log configuration file is used on both the
+client and the server.
+
+The `slider-client.xml` file is a hadoop-formatted XML options file, which
+is read by the Slider client -but not by they Slider Application Master.
+
+Here is an example file:
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>namenode:8033</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://namenode:9000</value>
+ </property>
+
+ <property>
+ <name>ipc.client.fallback-to-simple-auth-allowed</name>
+ <value>false</value>
+ </property>
+
+
+This defines both the filesystem and the YARN RM, and so obviates the need
+to declare either on the command line.
+
+If an option is defined in the `slider-client.xml` file and on the command line
+-be it by a `-D key=value` declaration or a `--manager` or `--filesystem`
+definition. (this holds even if the value is declared with `<final>true</final>`).
+
+## Selecting an alternate Slider configuration directory
+
+The environment variable `SLIDER_CONF_DIR` can be used to declare an alternate
+configuration directory. If set, the directory it identifies will be used
+as the source of the `log4j.properties` and `slider-client.xml` files.
+
+## Slider Client Configuration options
+
+As well as standard YARN and Hadoop configuration options, Slider supports
+a limited number of slider-specific configuration parameters.
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>localhost:2181,zookeeper2:4545</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue.priority</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>slider.yarn.restart.limit</name>
+ <value>5</value>
+ <description>How many times to start/restart the Slider AM</description>
+ </property>
+
+ <property>
+ <name>slider.cluster.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+ <property>
+ <name>slider.data.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+### `slider.zookeeper.quorum` - the zookeeper quorum.
+
+This defines the zookeeper quorum for this YARN cluster.
+
+It is used to locate the service registry, enable running instances to publish
+information about their application, and for clients to query this.
+
+It is also used as the default zookeeper binding for any application that
+uses zookeeper in its configuration -the value set when the application is
+defined will be copied into the instance definition file.
+
+### `"slider.registry.path"` - the zookeeper path for the service registry
+
+This declares the the zookeeper path for the service registry.
+
+### `slider.security.enabled` - enable security.
+
+This turns security on; consult [Security](/docs/security.html) for more information.
+
+
+### `slider.yarn.restart.limit` - set limit on Application Master Restarts
+
+This limits how many times YARN should start a failed application master.
+
+A short restart limit is useful when initially creating a cluster, as it
+ensures that YARN does not repeatedly try to restart a failing application.
+
+In production, however, a large number prevents YARN from halting a Slider
+application merely because failures in the underlying YARN cluster have
+triggered restarts.
+
+*Important:* The cluster-wide limit of `yarn.resourcemanager.am.max-attempts`
+places an upper limit on the number of retries that any application can request.
+If the application fails after less restarts than requested, check this cluster
+setting.
+
+### `slider.yarn.queue` - the name of the YARN queue for the cluster.
+
+This identifies the queue submit the application creation request to, which can
+define the priority, resource limits and other values of an application. All
+containers created in the Slider cluster will share this same queue.
+
+Default value: `default`.
+
+### `slider.yarn.queue.priority` - the name of the YARN queue for the cluster.
+
+This identifies the priority within the queue. The lower the value, the higher the
+priority
+
+Default value: `1`.
+
+ bin/slider thaw cl1 -D slider.yarn.queue.priority=5
+
+
+
+#### `slider.cluster.directory.permissions`
+
+An octal-format (`chmod`-style) permissions mask for the directory
+that contains the cluster specification `${user.home}/.slider/clusters/${clustername}`
+
+ <property>
+ <name>slider.cluster.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+#### `slider.data.directory.permissions`
+
+An octal-format (`chmod`-style) permissions mask for the directory
+that contains the application data `${user.home}/.slider/clusters/${clustername}/database`
+
+ <property>
+ <name>slider.data.directory.permissions</name>
+ <value>750</value>
+ </property>
+
+
+## Debugging configuration issues
+
+If the slider packages are set to log at debug level in the log4j configuration
+file, details on properties will be part of the copious output.
+
+
+## How client options are passed down to created clusters.
+
+Apart from the filesystem bindings, Client configuration options are
+not passed down to the XML site specification of the created cluster.
+
+The sole options passed down are the HDFS bindings: `fs.defaultFS`,
+which is passed down both as that property and as `fs.default.name`,
+and, in a secure cluster, the security flag (`slider.security.enabled`)
+and the HDFS Kerberos principal.
+
Propchange: incubator/slider/site/trunk/content/docs/client-configuration.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/docs/configuration/core.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/core.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/core.md (added)
+++ incubator/slider/site/trunk/content/docs/configuration/core.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,407 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Apache Slider Core Configuration Specification
+
+
+## Terminology
+
+
+*Application* A single application, such as an HBase cluster. An application
+is distribed across the YARN cluster.
+
+*Component* A single executable part of the larger application. An application
+may have multiple components, and multiple instances of each component.
+
+*YARN* Yet Another Resource Negotiator
+
+*YARN Resource Requirements* The requirements for a YARN resource request.
+Currently this consists of RAM and CPU requirements.
+
+*YARN Container*. An allocation portion of a servers resources granted
+to satisfy the requested YARN resource requirements. A process can be deployed
+to a container.
+
+
+*`resources.json`*: A file that describes the
+size of the application in terms of its component requirements: how many,
+and what their resource requirements are.
+
+*`application.json`*: A file that describes the
+size of the application in terms of its component requirements: how many,
+and what their resource requirements are.
+
+## Structure
+
+Configurations are stored in well-formed JSON files.
+1. Text MUST be saved in the UTF-8 format.
+1. Duplicate entries MUST NOT occur in any section.
+1. The ordering of elements is NOT significant.
+
+The JSON specification files all have a similar structure
+
+1. A `schema` string indicating version. Currently this is temporarily set to
+
+ "http://example.org/specification/v2.0.0"
+
+
+1. A global section, `/global` containing string properties
+1. A component section, `/components`.
+1. 0 or more sections under `/components` for each component, identified by component name,
+ containing string properties.
+1. 0 or 1 section `/metadata` containing arbitrary metadata (such as a description,
+author, or any other information that is not parsed or processed directly).
+
+
+The simplest valid specification file is
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ }
+ }
+
+
+## Property inheritance model and *resolution*
+
+
+There is a simple global to component inheritance model.
+
+1. Properties defined in `/global` define parameters across the entire application.
+1. Properties defined a section under `/components` define parameters for
+a specific component in the application.
+1. All global properties are propagated to each component.
+1. A component section may override any global property.
+1. The final set of configuration properties for a component is the global
+properties extended and overridden by the global set.
+1. The process of expanding the properties is termed *resolution*; the *resolved*
+specification is the outcome.
+1. There is NO form of explicitly cross-referencing another attribute. This
+MAY be added in future.
+1. There is NO sharing of information from the different `.json` files in a
+an application configuration.
+
+### Example
+
+Here is an example configuration
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden"
+
+ },
+ "worker": {
+ "name": "w",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+ }
+ }
+ }
+
+The `/global` section defines two properties
+
+ g1="a"
+ g2="b"
+
+These are the values visible to any part of the application which is
+not itself one of the components.
+
+
+There are three components defined, `simple`, `master` and `worker`.
+
+
+#### component `simple`:
+
+ g1="a"
+ g2="b"
+
+
+No settings have been defined specifically for the component; the global
+settings are applied.
+
+#### component `master`:
+
+ name="m",
+ g1="overridden"
+ g2="b"
+
+A new attribute, `name`, has been defined with the value `"m"`, and the
+global property `g1` has been overridden with the new value, `"overridden"`.
+The global property `g2` is passed down unchanged.
+
+
+#### component `worker`:
+
+ name="w",
+ g1="overridden-by-worker"
+ g2="b"
+ timeout: "1000"
+
+A new attribute, `name`, has been defined with the value `"w"`, and another,
+`timeout`, value "1000".
+
+The global property `g1` has been overridden with the new value, `"overridden-by-worker"`.
+
+The global property `g2` is passed down unchanged.
+
+This example shows some key points about the design
+
+* each component gets its own map of properties, which is independent from
+ that of other components.
+* all global properties are either present or overridden by a new value.
+ They can not be "undefined"
+* new properties defined in a component are not visible to any other component.
+
+The final *resolved* model is as follows
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden",
+ "g2": "b"
+ },
+ "worker": {
+ "name": "m",
+ "g1": "overridden-by-worker",
+ "g2": "b",
+ "timeout": "1000"
+ }
+ }
+ }
+
+This the specification JSON that would have generate exactly the same result as
+in the example, without any propagation of data from the global section
+to individual components.
+
+Note that a resolved specification can still have the resolution operation applied
+to it -it just does not have any effect.
+
+## Metadata
+
+The metadata section can contain arbitrary string values for use in diagnostics
+and by other applications.
+
+To avoid conflict with other applications, please use a unique name in strings,
+such as java-style package names.
+
+# Resource Requirements: `resources.json`
+
+This file declares the resource requirements for YARN for the components
+of an application.
+
+`instances`: the number of instances of a role desired.
+`yarn.vcores`: number of "virtual" required by a component.
+`yarn.memory`: the number of megabytes required by a component.
+
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5"
+ }
+ }
+ }
+
+The resolved file would be
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5",
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ }
+ }
+ }
+
+This declares this deployment of the application to consist of one instance of
+the master component, using 1 vcore and 1024MB of RAM, and five worker components
+each using one vcore and 512 MB of RAM.
+
+
+## Internal information, `internal.json`
+
+This contains internal data related to the deployment -it is not
+intended for manual editing.
+
+There MAY be a component, `diagnostics`. If defined, its content contains
+diagnostic information for support calls, and MUST NOT be interpreted
+during application deployment, (though it may be included in the generation
+of diagnostics reports)
+
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "Internal configuration DO NOT EDIT"
+ },
+ "global": {
+ "name": "small_cluster",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ },
+ "components": {
+
+ "diagnostics": {
+ "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ }
+ }
+ }
+
+
+## Deployment specification: `app_configuration.json`
+
+
+This defines parameters that are to be used when creating the instance of the
+application, and instances of the individual components.
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+ }
+
+The resolved specification defines the values that are passed to the
+different components.
+
+ {
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ }
+ }
+ }
+
+The `site.` properties have been passed down to each component, components
+whose templates may generate local site configurations. The override model
+does not prevent any component from overriding global configuration so as
+to create local configurations incompatible with the global state. (i.e.,
+there is no way to declare an attribute as final). It is the responsibility
+of the author of the configuration file (and their tools) to detect such issues.
Propchange: incubator/slider/site/trunk/content/docs/configuration/core.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/docs/configuration/example-app_configuration-resolved.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-app_configuration-resolved.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-app_configuration-resolved.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-app_configuration-resolved.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,42 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0",
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/example-app_configuration.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-app_configuration.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-app_configuration.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-app_configuration.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_small_cluster",
+ "zookeeper.hosts": "zoo1,zoo2,zoo3",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://cluster:8020",
+ "site.fs.default.name": "hdfs://cluster:8020",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+ },
+ "components": {
+
+ "worker": {
+ "jvm.heapsize": "512M"
+ },
+ "master": {
+ "jvm.heapsize": "512M"
+ }
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/example-empty.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-empty.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-empty.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-empty.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,8 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/example-internal.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-internal.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-internal.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-internal.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "Internal configuration DO NOT EDIT"
+ },
+ "global": {
+ "application.name": "small_cluster",
+ "application.type": "hbase",
+ "application": "hdfs://cluster:8020/apps/hbase/v/1.0.0/application.tar"
+ },
+ "components": {
+
+ "diagnostics": {
+ "create.hadoop.deployed.info": "(release-2.3.0) @dfe463",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT"
+ }
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/example-overridden-resolved.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-overridden-resolved.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-overridden-resolved.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-overridden-resolved.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden",
+ "g2": "b"
+ },
+ "worker": {
+ "name": "m",
+ "g1": "overridden-by-worker",
+ "g2": "b",
+ "timeout": "1000"
+ }
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/example-overridden.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-overridden.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-overridden.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-overridden.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,23 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ "g1": "a",
+ "g2": "b"
+ },
+ "components": {
+ "simple": {
+ },
+ "master": {
+ "name": "m",
+ "g1": "overridden"
+
+ },
+ "worker": {
+ "name": "m",
+ "g1": "overridden-by-worker",
+ "timeout": "1000"
+
+ }
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/example-resources.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/example-resources.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/example-resources.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/example-resources.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,25 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.vcores": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5",
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ }
+ }
+}
\ No newline at end of file
Added: incubator/slider/site/trunk/content/docs/configuration/index.md
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/index.md?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/index.md (added)
+++ incubator/slider/site/trunk/content/docs/configuration/index.md Tue Jun 24 19:46:37 2014
@@ -0,0 +1,38 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# Apache Slider: Specification of an application instance, revision 2.0
+
+The specification of an applicaton, comprises
+1. The persistent description of an application's configuration
+1. The persistent description of the desired topology and YARN resource
+requirements.
+1. The dynamic description of the running application, including information
+on the location of components and aggregated statistics.
+
+The specifics of this are covered in the [Core Configuration Specification](core.html)
+
+
+## Historical References
+
+1. [Specification](specification.html)
+1. [Redesign](redesign.html)
+
+
+1. [Example: current](original-hbase.json)
+1. [Example: proposed](proposed-hbase.json)
+
Propchange: incubator/slider/site/trunk/content/docs/configuration/index.md
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/trunk/content/docs/configuration/original-hbase.json
URL: http://svn.apache.org/viewvc/incubator/slider/site/trunk/content/docs/configuration/original-hbase.json?rev=1605167&view=auto
==============================================================================
--- incubator/slider/site/trunk/content/docs/configuration/original-hbase.json (added)
+++ incubator/slider/site/trunk/content/docs/configuration/original-hbase.json Tue Jun 24 19:46:37 2014
@@ -0,0 +1,139 @@
+{
+ "version": "1.0",
+ "name": "test_cluster_lifecycle",
+ "type": "hbase",
+ "state": 3,
+ "createTime": 1393512091276,
+ "updateTime": 1393512117286,
+ "originConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot",
+ "generatedConfigurationPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/generated",
+ "dataPath": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database",
+ "options": {
+ "zookeeper.port": "2181",
+ "site.hbase.master.startup.retainassign": "true",
+ "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz",
+ "site.fs.defaultFS": "hdfs://sandbox:8020",
+ "slider.container.failure.threshold": "5",
+ "site.fs.default.name": "hdfs://sandbox:8020",
+ "slider.cluster.directory.permissions": "0770",
+ "slider.am.monitoring.enabled": "false",
+ "zookeeper.path": "/yarnapps_slider_stevel_test_cluster_lifecycle",
+ "slider.tmp.dir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/tmp/am",
+ "slider.data.directory.permissions": "0770",
+ "zookeeper.hosts": "sandbox",
+ "slider.container.failure.shortlife": "60"
+ },
+ "info": {
+ "create.hadoop.deployed.info": "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
+ "create.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "create.hadoop.build.info": "2.3.0",
+ "create.time.millis": "1393512091276",
+ "create.time": "27 Feb 2014 14:41:31 GMT",
+ "slider.am.restart.supported": "false",
+ "live.time": "27 Feb 2014 14:41:56 GMT",
+ "live.time.millis": "1393512116881",
+ "status.time": "27 Feb 2014 14:42:08 GMT",
+ "status.time.millis": "1393512128726",
+ "yarn.vcores": "32",
+ "yarn.memory": "2048",
+ "status.application.build.info": "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "status.hadoop.build.info": "2.3.0",
+ "status.hadoop.deployed.info": "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
+ },
+ "statistics": {
+ "worker": {
+ "containers.start.started": 0,
+ "containers.live": 0,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 0,
+ "containers.requested": 0
+ },
+ "slider": {
+ "containers.unknown.completed": 0,
+ "containers.start.started": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+ },
+ "master": {
+ "containers.start.started": 0,
+ "containers.live": 0,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 0,
+ "containers.requested": 0
+ }
+ },
+ "status": {
+ },
+ "instances": {
+ "slider": [ "container_1393511571284_0002_01_000001" ]
+ },
+ "roles": {
+ "worker": {
+ "yarn.memory": "768",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.requested.instances": "0",
+ "role.name": "worker",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0",
+ "app.infoport": "0"
+ },
+ "slider": {
+ "yarn.memory": "256",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "1",
+ "role.requested.instances": "0",
+ "role.name": "slider",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "1",
+ "jvm.heapsize": "256M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ },
+ "master": {
+ "yarn.memory": "1024",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.requested.instances": "0",
+ "role.name": "master",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0",
+ "app.infoport": "0"
+ }
+ },
+ "clientProperties": {
+ "fs.defaultFS": "hdfs://sandbox:8020",
+ "hbase.cluster.distributed": "true",
+ "hbase.master.info.port": "0",
+ "hbase.master.port": "0",
+ "hbase.master.startup.retainassign": "true",
+ "hbase.regionserver.hlog.tolerable.lowreplication": "1",
+ "hbase.regionserver.info.port": "0",
+ "hbase.regionserver.port": "0",
+ "hbase.rootdir": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/database",
+ "hbase.tmp.dir": "./hbase-tmp",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.zookeeper.quorum": "sandbox",
+ "slider.template.origin": "hdfs://sandbox:8020/user/stevel/.slider/cluster/test_cluster_lifecycle/snapshot/hbase-site.xml",
+ "slider.unused.option": "1",
+ "zookeeper.znode.parent": "/yarnapps_slider_stevel_test_cluster_lifecycle"
+ }
+}