You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2014/05/30 18:47:16 UTC
svn commit: r1598665 [4/16] - in /incubator/slider/site/content: ./
architecture/ configuration/ css/ developing/ images/ images/logos/
images/profiles/ img/ js/ registry/ release_notes/ slider_specs/
specification/
Added: incubator/slider/site/content/configuration/specification.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/configuration/specification.html?rev=1598665&view=auto
==============================================================================
--- incubator/slider/site/content/configuration/specification.html (added)
+++ incubator/slider/site/content/configuration/specification.html Fri May 30 16:47:13 2014
@@ -0,0 +1,598 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2014-05-30
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20140530" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Apache Slider 0.30 (incubating) - </title>
+ <link rel="stylesheet" href="../css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="../css/site.css" />
+ <link rel="stylesheet" href="../css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="../js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+ </head>
+ <body class="topBarEnabled">
+
+
+
+
+
+ <div id="topbar" class="navbar navbar-fixed-top ">
+ <div class="navbar-inner">
+ <div class="container"><div class="nav-collapse">
+
+
+ <ul class="nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Documentation <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li class="dropdown-submenu">
+ <a href="../project-info.html" title="Project Information">Project Information</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../index.html" title="About">About</a>
+</li>
+ <li> <a href="../plugin-management.html" title="Plugin Management">Plugin Management</a>
+</li>
+ <li> <a href="../distribution-management.html" title="Distribution Management">Distribution Management</a>
+</li>
+ <li> <a href="../dependency-info.html" title="Dependency Information">Dependency Information</a>
+</li>
+ <li> <a href="../dependency-convergence.html" title="Dependency Convergence">Dependency Convergence</a>
+</li>
+ <li> <a href="../source-repository.html" title="Source Repository">Source Repository</a>
+</li>
+ <li> <a href="../mail-lists.html" title="Mailing Lists">Mailing Lists</a>
+</li>
+ <li> <a href="../issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
+</li>
+ <li> <a href="../integration.html" title="Continuous Integration">Continuous Integration</a>
+</li>
+ <li> <a href="../plugins.html" title="Project Plugins">Project Plugins</a>
+</li>
+ <li> <a href="../license.html" title="Project License">Project License</a>
+</li>
+ <li> <a href="../modules.html" title="Project Modules">Project Modules</a>
+</li>
+ <li> <a href="../dependency-management.html" title="Dependency Management">Dependency Management</a>
+</li>
+ <li> <a href="../team-list.html" title="Project Team">Project Team</a>
+</li>
+ <li> <a href="../project-summary.html" title="Project Summary">Project Summary</a>
+</li>
+ <li> <a href="../dependencies.html" title="Dependencies">Dependencies</a>
+</li>
+ </ul>
+ </li>
+
+ <li class="dropdown-submenu">
+ <a href="../project-reports.html" title="Project Reports">Project Reports</a>
+ <ul class="dropdown-menu">
+ <li> <a href="../surefire-report.html" title="Surefire Report">Surefire Report</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documents <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="../getting_started.html" title="Getting Started">Getting Started</a>
+</li>
+
+ <li> <a href="../manpage.html" title="manpage">manpage</a>
+</li>
+
+ <li> <a href="../troubleshooting.html" title="Troubleshooting">Troubleshooting</a>
+</li>
+
+ <li> <a href="../architecture/index.html" title="Architecture">Architecture</a>
+</li>
+
+ <li> <a href="../developing/index.html" title="Developing">Developing</a>
+</li>
+
+ <li> <a href="../exitcodes.html" title="Exitcodes">Exitcodes</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
+ <div class="container">
+ <div id="banner">
+ <div class="pull-left">
+ <a href=".././" id="bannerLeft">
+ <h2>Apache Slider (incubating)</h2>
+ </a>
+ </div>
+ <div class="pull-right"> <div id="bannerRight">
+ <img src="http://incubator.apache.org/images/apache-incubator-logo.png" />
+ </div>
+ </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li id="publishDate">Last Published: 2014-05-30</li>
+
+
+
+ <li id="projectVersion" class="pull-right">Version: 0.30</li>
+
+ </ul>
+ </div>
+
+
+
+ <div id="bodyColumn" >
+
+ <!-- -
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. --><h1>Apache Slider Specification of the “Cluster Description”</h1>
+
+<ul>
+
+<li>This is partially obsolete. Slider still returns the Slider Cluster Description as changing it will break most of the unit tests -once these are updated this document will be completely obsolete and replaced with a new one.</li>
+</ul>
+<div class="section">
+<div class="section">
+<h3>Notation:<a name="Notation:"></a></h3>
+<p>In this document, a full path to a value is represented as a path <tt>options/zookeeper.port</tt> ; an assigment as <tt>options/zookeeper.port=2181</tt>.</p>
+<p>A wildcard indicates all entries matching a path: <tt>options/zookeeper.*</tt> or <tt>/roles/*/yarn.memory</tt></p></div></div>
+<div class="section">
+<h2>Core Concepts<a name="Core_Concepts"></a></h2>
+<p>The specificaton of an application instance is defined in an application instance directory, <tt>${user.home}/.slidera/clusters/${clustername}/cluster.json</tt>)</p></div>
+<div class="section">
+<h2>Sections for specifying and describing cluster state<a name="Sections_for_specifying_and_describing_cluster_state"></a></h2>
+<p>The cluster desciption is hierarchal, with standardized sections.</p>
+<p>Different sections have one of three roles.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>
+<p>Storage and specification of internal properties used to define a cluster -properties that should not be modified by users -doing so is likely to render the cluster undeployable.</p></li>
+
+<li>
+<p>Storage and specification of the components deployed by Slider. These sections define options for the deployed application, the size of the deployed application, attributes of the deployed roles, and customizable aspects of the Slider application master. </p></li>
+</ol>
+<p>This information defines the <i>desired state</i> of a cluster.</p>
+<p>Users may edit these sections, either via the CLI, or by directly editing the <tt>cluster.json</tt> file of a frozen cluster.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>Status information provided by a running cluster. These include: information about the cluster, statistics, information about reach role in the cluster -as well as other aspects of the deployment.</li>
+</ol>
+<p>This information describes the <i>actual state</i> of a cluster.</p>
+<p>Using a common format for both the specification and description of a cluster may be confusing, but it is designed to unify the logic needed to parse and process cluster descriptions. There is only one JSON file to parse -merely different sections of relevance at different times.</p></div>
+<div class="section">
+<h2>Role-by-role subsections<a name="Role-by-role_subsections"></a></h2>
+<p>A slider-deployed application consists of the single Slider application master, and one or more roles -specific components in the actual application.</p>
+<p>The <tt>/roles</tt> section contains a listing for each role, declaring the number of instances of each role desired, possibly along with some details defining the actual execution of the application.</p>
+<p>The <tt>/statistics/roles/</tt> section returns statistics on each role, while <tt>/instances</tt> has a per-role entry listing the YARN containers hosting instances. </p></div>
+<div class="section">
+<h2>Cluster information for applications<a name="Cluster_information_for_applications"></a></h2>
+<p>The AM/application provider may generate information for use by client applications.</p>
+<p>There are three ways to provide this</p>
+
+<ol style="list-style-type: decimal">
+
+<li>A section in which simple key-value pairs are provided for interpretation by client applications -usually to generate configuration documents</li>
+
+<li>A listing of files that may be provided directly to a client. The API to provide these files is not covered by this document.</li>
+
+<li>A provider-specific section in which arbitrary values and structures may be defined. This allows greater flexibility in the information that a provider can publish -though it does imply custom code to process this data on the client.</li>
+</ol>
+<h1>Persistent Specification Sections</h1></div>
+<div class="section">
+<h2>“/” : root<a name="a_:_root"></a></h2>
+<p>The root contains a limited number of key-value pairs, </p>
+
+<ul>
+
+<li>
+<p><tt>version</tt>: string; required. The version of the JSON file, as an <tt>x.y.z</tt> version string.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>Applications MUST verify that they can process a specific version.</li>
+
+<li>The version number SHOULD be incremented in the minor “z” value after enhancements that are considered backwards compatible. Incompatible updates MUST be updated with a new “y” value. The final, “x” number, is to be reserved for major reworkings of the cluster specification itself (this document or its successors).</li>
+ </ol></li>
+
+<li>
+<p><tt>name</tt>: string; required. Cluster name; </p></li>
+
+<li><tt>type</tt>: string; required. Reference to the provider type -this triggers a Hadoop configuration property lookup to find the implementation classes.</li>
+
+<li><tt>valid</tt>: boolean; required. Flag to indicate whether or not a specification is considered valid. If false, the rest of the document is in an unknown state.</li>
+</ul></div>
+<div class="section">
+<h2><tt>/slider-internal</tt>: internal confiugration<a name="aslider-internal:_internal_confiugration"></a></h2>
+<p>Stores internal configuration options. These parameters are not defined in this document.</p></div>
+<div class="section">
+<h2><tt>/diagnostics</tt>: diagnostics sections<a name="adiagnostics:_diagnostics_sections"></a></h2>
+<p>Persisted list of information about Slider. </p>
+<p>Static information about the file history</p>
+
+<div class="source">
+<pre>"diagnostics" : {
+ "create.hadoop.deployed.info" :
+ "(detached from release-2.3.0) @dfe46336fbc6a044bc124392ec06b85",
+ "create.application.build.info" :
+ "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+ "create.hadoop.build.info" : "2.3.0",
+ "create.time.millis" : "1393512091276",
+},
+</pre></div>
+<p>This information is not intended to provide anything other than diagnostics to an application; the values and their meaning are not defined. All applications MUST be able to process an empty or absent <tt>/diagnostics</tt> section.</p></div>
+<div class="section">
+<h2>Options: cluster options<a name="Options:_cluster_options"></a></h2>
+<p>A persisted list of options used by Slider and its providers to build up the AM and the configurations of the deployed service components</p>
+
+<div class="source">
+<pre>"options": {
+ "slider.am.monitoring.enabled": "false",
+ "slider.cluster.application.image.path": "hdfs://sandbox:8020/hbase.tar.gz",
+ "slider.container.failure.threshold": "5",
+ "slider.container.failure.shortlife": "60",
+ "zookeeper.port": "2181",
+ "zookeeper.path": "/yarnapps_slider_stevel_test_cluster_lifecycle",
+ "zookeeper.hosts": "sandbox",
+ "site.hbase.master.startup.retainassign": "true",
+ "site.fs.defaultFS": "hdfs://sandbox:8020",
+ "site.fs.default.name": "hdfs://sandbox:8020",
+ "env.MALLOC_ARENA_MAX": "4",
+ "site.hbase.master.info.port": "0",
+ "site.hbase.regionserver.info.port": "0"
+},
+</pre></div>
+<p>Many of the properties are automatically set by Slider when a cluster is constructed. They may be edited afterwards.</p>
+<div class="section">
+<h3>Standard Option types<a name="Standard_Option_types"></a></h3>
+<p>All option values MUST be strings.</p>
+<div class="section">
+<h4><tt>slider.</tt><a name="slider."></a></h4>
+<p>All options that begin with <tt>slider.</tt> are intended for use by slider and providers to configure the Slider application master itself, and the application. For example, <tt>slider.container.failure.threshold</tt> defines the number of times a container must fail before the role (and hence the cluster) is considered to have failed. As another example, the zookeeper bindings such as <tt>zookeeper.hosts</tt> are read by the HBase and Ambari providers, and used to modify the applications’ site configurations with application-specific properties.</p></div>
+<div class="section">
+<h4><tt>site.</tt><a name="site."></a></h4>
+<p>These are properties that are expected to be propagated to an application’s <tt>site</tt> configuration -if such a configuration is created. For HBase, the site file is <tt>hbase-site.xml</tt>; for Accumulo it is <tt>accumulo-site.xml</tt></p>
+
+<ol style="list-style-type: decimal">
+
+<li>The destination property is taken by removing the prefix <tt>site.</tt>, and setting the shortened key with the defined value.</li>
+
+<li>Not all applications have the notion of a site file; These applications MAY ignore the settings.</li>
+
+<li>Providers MAY validate site settings to recognise invalid values. This aids identifying and diagnosing startup problems.</li>
+</ol></div>
+<div class="section">
+<h4><tt>env.</tt><a name="env."></a></h4>
+<p>These are options to configure environment variables in the roles. When a container is started, all <tt>env.</tt> options have the prefix removed, and are then set as environment variables in the target context.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>The Slider AM uses these values to configure itself, after following the option/role merge process.</li>
+
+<li>Application providers SHOULD follow the same process.</li>
+</ol></div></div></div>
+<div class="section">
+<h2>‘/roles’: role declarations<a name="aroles:_role_declarations"></a></h2>
+<p>The <tt>/roles/$ROLENAME/</tt> clauses each provide options for a specific role.</p>
+<p>This includes 1. <tt>role.instances</tt>: defines the number of instances of a role to create 1. <tt>env.</tt> environment variables for launching the container 1. <tt>yarn.</tt> properties to configure YARN requests. 1. <tt>jvm.heapsize</tt>: an option supported by some providers to fix the heap size of a component.</p>
+
+<div class="source">
+<pre> "worker": {
+ "yarn.memory": "768",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "0",
+ "role.name": "worker",
+ "role.failed.starting.instances": "0",
+ "jvm.heapsize": "512M",
+ "yarn.vcores": "1",
+ },
+</pre></div>
+<p>The role <tt>slider</tt> represents the Slider Application Master itself.</p>
+
+<div class="source">
+<pre> "slider": {
+ "yarn.memory": "256",
+ "env.MALLOC_ARENA_MAX": "4",
+ "role.instances": "1",
+ "role.name": "slider",
+ "jvm.heapsize": "256M",
+ "yarn.vcores": "1",
+ },
+</pre></div>
+<p>Providers may support a fixed number of roles -or they may support a dynamic number of roles defined at run-time, potentially from other data sources.</p></div>
+<div class="section">
+<h2>How <tt>/options</tt> and role options are merged.<a name="How_options_and_role_options_are_merged."></a></h2>
+<p>The options declared for a specific role are merged with the cluster-wide options to define the final options for a role. This is implemented in a simple override model: role-specific options can override any site-wide options.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>The options defined in <tt>/options</tt> are used to create the initial option map for each role.</li>
+
+<li>The role’s options are then applied to the map -this may overwrite definitions from the <tt>/options</tt> section.</li>
+
+<li>There is no way to “undefine” a cluster option, merely overwrite it.</li>
+
+<li>The merged map is then used by the provider to create the component.</li>
+
+<li>The special <tt>slider</tt> role is used in the CLI to define the attributes of the AM.</li>
+</ol>
+<p>Options set on a role do not affect any site-wide options: they are specific to the invidual role being created. </p>
+<p>As such, overwriting a <tt>site.</tt> option may have no effect -or it it may change the value of a site configuration document <i>in that specific role instance</i>.</p>
+<div class="section">
+<h3>Standard role options<a name="Standard_role_options"></a></h3>
+
+<ul>
+
+<li><tt>role.instances</tt> : number; required. The number of instances of that role desired in the application.</li>
+
+<li><tt>yarn.vcores</tt> : number. The number of YARN “virtual cores” to request for each role instance. The larger the number, the more CPU allocation -and potentially the longer time to satisfy the request and so instantiate the node. If the value ’“-1”<tt>is used -for any role but</tt>slider`-the maximum value available to the application is requested.</li>
+
+<li><tt>yarn.memory</tt> : number. The number in Megabytes of RAM to request for each role instance. The larger the number, the more memory allocation -and potentially the longer time to satisfy the request and so instantiate the node. If the value ’“-1”<tt>is used -for any role but</tt>slider`-the maximum value available to the application is requested.</li>
+
+<li>
+<p><tt>env.</tt> environment variables. String environment variables to use when setting up the container</p></li>
+</ul></div>
+<div class="section">
+<h3>Provider-specific role options<a name="Provider-specific_role_options"></a></h3>
+
+<ul>
+
+<li><tt>jvm.heapsize</tt> -the amount of memory for a provider to allocate for a processes JVM. Example “512M”. This option MAY be implemented by a provider.</li>
+</ul>
+<h1>Dynamic Information Sections</h1>
+<p>These are the parts of the document that provide dynamic run-time information about an application. They are provided by the Slider Application Master when a request for the cluster status is issued.</p></div></div>
+<div class="section">
+<h2><tt>/info</tt><a name="ainfo"></a></h2>
+<p>Dynamic set of string key-value pairs containing information about the running application -as provided by th </p>
+<p>The values in this section are not normatively defined. </p>
+<p>Here are some standard values</p>
+
+<ul>
+
+<li>
+<p><tt>slider.am.restart.supported"</tt> whether the AM supports service restart without killing all the containers hosting the role instances:</p>
+
+<div class="source">
+<pre>"slider.am.restart.supported" : "false",
+</pre></div></li>
+
+<li>
+<p>timestamps of the cluster going live, and when the status query was made</p>
+
+<div class="source">
+<pre>"live.time" : "27 Feb 2014 14:41:56 GMT",
+"live.time.millis" : "1393512116881",
+"status.time" : "27 Feb 2014 14:42:08 GMT",
+"status.time.millis" : "1393512128726",
+</pre></div></li>
+
+<li>
+<p>yarn data provided to the AM</p>
+
+<div class="source">
+<pre>"yarn.vcores" : "32",
+"yarn.memory" : "2048",
+</pre></div></li>
+
+<li>
+<p>information about the application and hadoop versions in use. Here the application was built using Hadoop 2.3.0, but is running against the version of Hadoop built for HDP-2.</p>
+
+<div class="source">
+<pre>"status.application.build.info" : "Slider Core-0.13.0-SNAPSHOT Built against commit# 1a94ee4aa1 on Java 1.7.0_45 by stevel",
+"status.hadoop.build.info" : "2.3.0",
+"status.hadoop.deployed.info" : "bigwheel-m16-2.2.0 @704f1e463ebc4fb89353011407e965"
+</pre></div></li>
+</ul>
+<p>As with the <tt>/diagnostics</tt> section, this area is primarily intended for debugging.</p>
+<p>## <tt>/instances</tt>: instance list</p>
+<p>Information about the live containers in a cluster</p>
+
+<div class="source">
+<pre> "instances": {
+ "slider": [ "container_1393511571284_0002_01_000001" ],
+ "master": [ "container_1393511571284_0002_01_000003" ],
+ "worker": [
+ "container_1393511571284_0002_01_000002",
+ "container_1393511571284_0002_01_000004"
+ ]
+ },
+</pre></div></div>
+<div class="section">
+<h2><tt>/status</tt>: detailed dynamic state<a name="astatus:_detailed_dynamic_state"></a></h2>
+<p>This provides more detail on the application including live and failed instances</p>
+<div class="section">
+<h3><tt>/status/live</tt>: live role instances by container<a name="astatuslive:_live_role_instances_by_container"></a></h3>
+
+<div class="source">
+<pre>"cluster": {
+ "live": {
+ "worker": {
+ "container_1394032374441_0001_01_000003": {
+ "name": "container_1394032374441_0001_01_000003",
+ "role": "worker",
+ "roleId": 1,
+ "createTime": 1394032384451,
+ "startTime": 1394032384503,
+ "released": false,
+ "host": "192.168.1.88",
+ "state": 3,
+ "exitCode": 0,
+ "command": "hbase-0.98.0/bin/hbase --config $PROPAGATED_CONFDIR regionserver start 1><LOG_DIR>/region-server.txt 2>&1 ; ",
+ "diagnostics": "",
+ "environment": [
+ "HADOOP_USER_NAME=\"slider\"",
+ "HBASE_LOG_DIR=\"/tmp/slider-slider\"",
+ "HBASE_HEAPSIZE=\"256\"",
+ "MALLOC_ARENA_MAX=\"4\"",
+ "PROPAGATED_CONFDIR=\"$PWD/propagatedconf\""
+ ]
+ }
+ }
+ failed : {}
+ }
+</pre></div>
+<p>All live instances MUST be described in <tt>/status/live</tt></p>
+<p>Failed clusters MAY be listed in the <tt>/status/failed</tt> section, specifically, a limited set of recently failed clusters SHOULD be provided.</p>
+<p>Future versions of this document may introduce more sections under <tt>/status</tt>.</p></div>
+<div class="section">
+<h3><tt>/status/rolestatus</tt>: role status information<a name="astatusrolestatus:_role_status_information"></a></h3>
+<p>This lists the current status of the roles: How many are running vs requested, how many are being released.</p>
+
+<div class="source">
+<pre>"rolestatus": {
+ "worker": {
+ "role.instances": "2",
+ "role.requested.instances": "0",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "2",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "1"
+ },
+ "slider": {
+ "role.instances": "1",
+ "role.requested.instances": "0",
+ "role.name": "slider",
+ "role.actual.instances": "1",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ },
+ "master": {
+ "role.instances": "1",
+ "role.requested.instances": "1",
+ "role.name": "master",
+ "role.failed.starting.instances": "0",
+ "role.actual.instances": "0",
+ "role.releasing.instances": "0",
+ "role.failed.instances": "0"
+ }
+}
+</pre></div></div>
+<div class="section">
+<h3><tt>/status/provider</tt>: provider-specific information<a name="astatusprovider:_provider-specific_information"></a></h3>
+<p>Providers MAY publish information to the <tt>/status/provider</tt> section.</p>
+
+<ol style="list-style-type: decimal">
+
+<li>There’s no restriction on what JSON is permitted in this section.</li>
+
+<li>Providers may make their own updates to the application state to read and write this block -operations that are asynchronous to any status queries.</li>
+</ol></div></div>
+<div class="section">
+<h2><tt>/statistics</tt>: aggregate statistics<a name="astatistics:_aggregate_statistics"></a></h2>
+<p>Statistics on the cluster and each role in the cluster </p>
+<p>Better to have a specific <tt>/statistics/cluster</tt> element, and to move the roles’ statistics under <tt>/statistics/roles</tt>:</p>
+
+<div class="source">
+<pre>"statistics": {
+ "cluster": {
+ "containers.unknown.completed": 0,
+ "containers.start.completed": 3,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+ },
+ "roles": {
+ "worker": {
+ "containers.start.completed": 0,
+ "containers.live": 2,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 2,
+ "containers.requested": 0
+ },
+ "master": {
+ "containers.start.completed": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 1,
+ "containers.requested": 0
+ }
+ }
+},
+</pre></div>
+<p><tt>/statistics/cluster</tt> provides aggregate statistics for the entire cluster.</p>
+<p>Under <tt>/statistics/roles</tt> MUST come an entry for each role in the cluster.</p>
+<p>All simple values in statistics section are integers.</p>
+<div class="section">
+<h3><tt>/clientProperties</tt><a name="aclientProperties"></a></h3>
+<p>The <tt>/clientProperties</tt> section contains key-val pairs of type string, the expectation being this is where providers can insert specific single attributes for client applications.</p>
+<p>These values can be converted to application-specific files on the client, in code -as done today in the Slider CLI-, or via template expansion (beyond the scope of this document.</p></div>
+<div class="section">
+<h3><tt>/clientfiles</tt><a name="aclientfiles"></a></h3>
+<p>This section list all files that an application instance MAY generate for clients, along with with a description.</p>
+
+<div class="source">
+<pre>"/clientfiles/hbase-site.xml": "site information for HBase"
+"/clientfiles/log4.properties": "log4.property file"
+</pre></div>
+<p>Client configuration file retrieval is by other means; this status operation merely lists files that are available;</p></div></div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container">
+ <div class="row span12">Copyright © 2014
+ <a href="http://www.apache.org/">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+ <?xml version="1.0" encoding="UTF-8"?>
+<div class="row-fluid">Apache Slider, Slider, Apache, and the Apache Incubator logo are trademarks of The Apache Software Foundation.</div>
+
+ <p id="poweredBy" class="pull-right">
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="../images/logos/maven-feather.png" />
+ </a>
+ </p>
+
+ </div>
+ </footer>
+ </body>
+</html>
Propchange: incubator/slider/site/content/configuration/specification.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/css/apache-maven-fluido-1.3.0.min.css
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/css/apache-maven-fluido-1.3.0.min.css?rev=1598665&view=auto
==============================================================================
--- incubator/slider/site/content/css/apache-maven-fluido-1.3.0.min.css (added)
+++ incubator/slider/site/content/css/apache-maven-fluido-1.3.0.min.css Fri May 30 16:47:13 2014
@@ -0,0 +1,9 @@
+/*!
+ * Bootstrap v2.1.0
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
[... 3 lines stripped ...]
Propchange: incubator/slider/site/content/css/apache-maven-fluido-1.3.0.min.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/css/print.css
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/css/print.css?rev=1598665&view=auto
==============================================================================
--- incubator/slider/site/content/css/print.css (added)
+++ incubator/slider/site/content/css/print.css Fri May 30 16:47:13 2014
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/* $Id: print.css 1201871 2011-11-14 20:18:24Z simonetripodi $ */
+
+#banner, #footer, #leftcol, #breadcrumbs, .docs #toc, .docs .courtesylinks, #leftColumn, #navColumn {display: none !important;}
+#bodyColumn, body.docs div.docs {margin: 0 !important;border: none !important}
Propchange: incubator/slider/site/content/css/print.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/css/site.css
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/css/site.css?rev=1598665&view=auto
==============================================================================
--- incubator/slider/site/content/css/site.css (added)
+++ incubator/slider/site/content/css/site.css Fri May 30 16:47:13 2014
@@ -0,0 +1 @@
+/* You can override this file with your own styles */
\ No newline at end of file
Propchange: incubator/slider/site/content/css/site.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/debugging.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/debugging.html?rev=1598665&view=auto
==============================================================================
--- incubator/slider/site/content/debugging.html (added)
+++ incubator/slider/site/content/debugging.html Fri May 30 16:47:13 2014
@@ -0,0 +1,239 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2014-05-30
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20140530" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Apache Slider 0.30 (incubating) - </title>
+ <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="./css/site.css" />
+ <link rel="stylesheet" href="./css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+ </head>
+ <body class="topBarEnabled">
+
+
+
+
+
+ <div id="topbar" class="navbar navbar-fixed-top ">
+ <div class="navbar-inner">
+ <div class="container"><div class="nav-collapse">
+
+
+ <ul class="nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Documentation <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li class="dropdown-submenu">
+ <a href="project-info.html" title="Project Information">Project Information</a>
+ <ul class="dropdown-menu">
+ <li> <a href="index.html" title="About">About</a>
+</li>
+ <li> <a href="plugin-management.html" title="Plugin Management">Plugin Management</a>
+</li>
+ <li> <a href="distribution-management.html" title="Distribution Management">Distribution Management</a>
+</li>
+ <li> <a href="dependency-info.html" title="Dependency Information">Dependency Information</a>
+</li>
+ <li> <a href="dependency-convergence.html" title="Dependency Convergence">Dependency Convergence</a>
+</li>
+ <li> <a href="source-repository.html" title="Source Repository">Source Repository</a>
+</li>
+ <li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
+</li>
+ <li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
+</li>
+ <li> <a href="integration.html" title="Continuous Integration">Continuous Integration</a>
+</li>
+ <li> <a href="plugins.html" title="Project Plugins">Project Plugins</a>
+</li>
+ <li> <a href="license.html" title="Project License">Project License</a>
+</li>
+ <li> <a href="modules.html" title="Project Modules">Project Modules</a>
+</li>
+ <li> <a href="dependency-management.html" title="Dependency Management">Dependency Management</a>
+</li>
+ <li> <a href="team-list.html" title="Project Team">Project Team</a>
+</li>
+ <li> <a href="project-summary.html" title="Project Summary">Project Summary</a>
+</li>
+ <li> <a href="dependencies.html" title="Dependencies">Dependencies</a>
+</li>
+ </ul>
+ </li>
+
+ <li class="dropdown-submenu">
+ <a href="project-reports.html" title="Project Reports">Project Reports</a>
+ <ul class="dropdown-menu">
+ <li> <a href="surefire-report.html" title="Surefire Report">Surefire Report</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documents <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="getting_started.html" title="Getting Started">Getting Started</a>
+</li>
+
+ <li> <a href="manpage.html" title="manpage">manpage</a>
+</li>
+
+ <li> <a href="troubleshooting.html" title="Troubleshooting">Troubleshooting</a>
+</li>
+
+ <li> <a href="architecture/index.html" title="Architecture">Architecture</a>
+</li>
+
+ <li> <a href="developing/index.html" title="Developing">Developing</a>
+</li>
+
+ <li> <a href="exitcodes.html" title="Exitcodes">Exitcodes</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
+ <div class="container">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="./" id="bannerLeft">
+ <h2>Apache Slider (incubating)</h2>
+ </a>
+ </div>
+ <div class="pull-right"> <div id="bannerRight">
+ <img src="http://incubator.apache.org/images/apache-incubator-logo.png" />
+ </div>
+ </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li id="publishDate">Last Published: 2014-05-30</li>
+
+
+
+ <li id="projectVersion" class="pull-right">Version: 0.30</li>
+
+ </ul>
+ </div>
+
+
+
+ <div id="bodyColumn" >
+
+ <!-- -
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. --><h1>Debugging Apache Slider</h1>
+<p>There are a number of options available to you for debugging Slider applications. They include:</p>
+
+<ul>
+
+<li>Using Slider logging</li>
+
+<li>IDE-based remote debugging of the Application Master</li>
+</ul>
+<div class="section">
+<h2>Using Slider logging<a name="Using_Slider_logging"></a></h2>
+<p>There are a number of options for viewing the generated log files:</p>
+
+<ol style="list-style-type: decimal">
+
+<li>Using a web browser</li>
+
+<li>Accessing the host machine</li>
+</ol>
+<div class="section">
+<h3>Using a web browser<a name="Using_a_web_browser"></a></h3>
+<p>The log files are accessible via the Yarn Resource Manager UI. From the main page (e.g. <tt>http://${YARN_RESOURCE_MGR_HOST}:8088</tt>), click on the link for the application instance of interest, and then click on the <tt>logs</tt> link. This will present you with a page with links to the <tt>slider-err.txt</tt> file and the <tt>slider-out.txt</tt> file. The former is the file you should select -it is where the applicaton logs go Once the log page is presented, click on the link at the top of the page (“Click here for full log”) to view the entire file.</p>
+<p>If the file <tt>slider-out.txt</tt> is empty, then examine <tt>slider-err.txt</tt> -an empty output log usually means that the java process failed to start -this should be logged in the error file.</p></div>
+<div class="section">
+<h3>Accessing the host machine<a name="Accessing_the_host_machine"></a></h3>
+<p>If access to other log files is required, there is the option of logging in to the host machine on which the application component is running -provided you have the correct permissions.</p>
+<p>The root directory for all YARN associated files is the value of <tt>yarn.nodemanager.log-dirs</tt> in <tt>yarn-site.xml</tt> - e.g. <tt>/hadoop/yarn/log</tt>. Below the root directory you will find an application and container sub-directory (e.g. <tt>/application_1398372047522_0009/container_1398372047522_0009_01_000001/</tt>). Below the container directory you will find any log files associated with the processes running in the given Yarn container.</p>
+<p>Within a container log the following files are useful while debugging the application.</p>
+<p><b>agent.log</b> </p>
+<p>E.g. <tt>application_1398098639743_0024/container_1398098639743_0024_01_000003/infra/log/agent.log</tt> This file contains the logs from the Slider-Agent.</p>
+<p><b>application component log</b></p>
+<p>E.g. <tt>./log/application_1398098639743_0024/container_1398098639743_0024_01_000003/app/log/hbase-yarn-regionserver-c6403.ambari.apache.org.log</tt></p>
+<p>The location of the application log is defined by the application. “${AGENT_LOG_ROOT}” is a symbol available to the app developers to use as a root folder for logging.</p>
+<p><b>agent operations log</b></p>
+<p>E.g. ./log/application_1398098639743_0024/container_1398098639743_0024_01_000003/app/command-log/</p>
+<p>The command logs produced by the slider-agent are available in the <tt>command-log</tt> folder relative to <tt>${AGENT_LOG_ROOT}/app</tt></p>
+<p>Note that the <i>fish</i> shell is convenient for debugging, as <tt>cat log/**/slider-out.txt</tt> will find the relevant output file irrespective of what the path leading to it is.</p></div></div>
+<div class="section">
+<h2>IDE-based remote debugging of the Application Master<a name="IDE-based_remote_debugging_of_the_Application_Master"></a></h2>
+<p>For situations in which the logging does not yield enough information to debug an issue, the user has the option of specifying JVM command line options for the Application Master that enable attaching to the running process with a debugger (e.g. the remote debugging facilities in Eclipse or Intellij IDEA). In order to specify the JVM options, edit the application configuration file (the file specified as the <tt>--template</tt> argument value on the command line for cluster creation) and specify the <tt>jvm.opts</tt> property for the <tt>slider-appmaster</tt> component:</p>
+
+<div class="source">
+<pre>`"components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M",
+ "jvm.opts": "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
+ },
+</pre></div>
+<p> …`</p>
+<p>You may specify <tt>suspend=y</tt> in the line above if you wish to have the application master process wait for the debugger to attach before beginning its processing.</p></div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container">
+ <div class="row span12">Copyright © 2014
+ <a href="http://www.apache.org/">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+ <?xml version="1.0" encoding="UTF-8"?>
+<div class="row-fluid">Apache Slider, Slider, Apache, and the Apache Incubator logo are trademarks of The Apache Software Foundation.</div>
+
+ <p id="poweredBy" class="pull-right">
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
+ </a>
+ </p>
+
+ </div>
+ </footer>
+ </body>
+</html>
Propchange: incubator/slider/site/content/debugging.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/dependencies.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/dependencies.html?rev=1598665&view=auto
==============================================================================
--- incubator/slider/site/content/dependencies.html (added)
+++ incubator/slider/site/content/dependencies.html Fri May 30 16:47:13 2014
@@ -0,0 +1,179 @@
+<!DOCTYPE html>
+<!--
+ | Generated by Apache Maven Doxia at 2014-05-30
+ | Rendered using Apache Maven Fluido Skin 1.3.0
+-->
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+ <meta charset="UTF-8" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ <meta name="Date-Revision-yyyymmdd" content="20140530" />
+ <meta http-equiv="Content-Language" content="en" />
+ <title>Apache Slider 0.30 (incubating) - Project Dependencies</title>
+ <link rel="stylesheet" href="./css/apache-maven-fluido-1.3.0.min.css" />
+ <link rel="stylesheet" href="./css/site.css" />
+ <link rel="stylesheet" href="./css/print.css" media="print" />
+
+
+ <script type="text/javascript" src="./js/apache-maven-fluido-1.3.0.min.js"></script>
+
+
+ </head>
+ <body class="topBarEnabled">
+
+
+
+
+
+ <div id="topbar" class="navbar navbar-fixed-top ">
+ <div class="navbar-inner">
+ <div class="container"><div class="nav-collapse">
+
+
+ <ul class="nav">
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project Documentation <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li class="dropdown-submenu">
+ <a href="project-info.html" title="Project Information">Project Information</a>
+ <ul class="dropdown-menu">
+ <li> <a href="index.html" title="About">About</a>
+</li>
+ <li> <a href="plugin-management.html" title="Plugin Management">Plugin Management</a>
+</li>
+ <li> <a href="distribution-management.html" title="Distribution Management">Distribution Management</a>
+</li>
+ <li> <a href="dependency-info.html" title="Dependency Information">Dependency Information</a>
+</li>
+ <li> <a href="dependency-convergence.html" title="Dependency Convergence">Dependency Convergence</a>
+</li>
+ <li> <a href="source-repository.html" title="Source Repository">Source Repository</a>
+</li>
+ <li> <a href="mail-lists.html" title="Mailing Lists">Mailing Lists</a>
+</li>
+ <li> <a href="issue-tracking.html" title="Issue Tracking">Issue Tracking</a>
+</li>
+ <li> <a href="integration.html" title="Continuous Integration">Continuous Integration</a>
+</li>
+ <li> <a href="plugins.html" title="Project Plugins">Project Plugins</a>
+</li>
+ <li> <a href="license.html" title="Project License">Project License</a>
+</li>
+ <li> <a href="modules.html" title="Project Modules">Project Modules</a>
+</li>
+ <li> <a href="dependency-management.html" title="Dependency Management">Dependency Management</a>
+</li>
+ <li> <a href="team-list.html" title="Project Team">Project Team</a>
+</li>
+ <li> <a href="project-summary.html" title="Project Summary">Project Summary</a>
+</li>
+ <li> <a href="dependencies.html" title="Dependencies">Dependencies</a>
+</li>
+ </ul>
+ </li>
+
+ <li class="dropdown-submenu">
+ <a href="project-reports.html" title="Project Reports">Project Reports</a>
+ <ul class="dropdown-menu">
+ <li> <a href="surefire-report.html" title="Surefire Report">Surefire Report</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Documents <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+ <li> <a href="getting_started.html" title="Getting Started">Getting Started</a>
+</li>
+
+ <li> <a href="manpage.html" title="manpage">manpage</a>
+</li>
+
+ <li> <a href="troubleshooting.html" title="Troubleshooting">Troubleshooting</a>
+</li>
+
+ <li> <a href="architecture/index.html" title="Architecture">Architecture</a>
+</li>
+
+ <li> <a href="developing/index.html" title="Developing">Developing</a>
+</li>
+
+ <li> <a href="exitcodes.html" title="Exitcodes">Exitcodes</a>
+</li>
+ </ul>
+ </li>
+ </ul>
+
+
+
+
+ </div>
+
+ </div>
+ </div>
+ </div>
+
+ <div class="container">
+ <div id="banner">
+ <div class="pull-left">
+ <a href="./" id="bannerLeft">
+ <h2>Apache Slider (incubating)</h2>
+ </a>
+ </div>
+ <div class="pull-right"> <div id="bannerRight">
+ <img src="http://incubator.apache.org/images/apache-incubator-logo.png" />
+ </div>
+ </div>
+ <div class="clear"><hr/></div>
+ </div>
+
+ <div id="breadcrumbs">
+ <ul class="breadcrumb">
+
+
+ <li id="publishDate">Last Published: 2014-05-30</li>
+
+
+
+ <li id="projectVersion" class="pull-right">Version: 0.30</li>
+
+ </ul>
+ </div>
+
+
+
+ <div id="bodyColumn" >
+
+ <a name="Project_Dependencies"></a>
+<div class="section">
+<h2>Project Dependencies<a name="Project_Dependencies"></a></h2>
+<p>There are no dependencies for this project. It is a standalone application that does not depend on any other project.</p></div>
+ </div>
+ </div>
+
+ <hr/>
+
+ <footer>
+ <div class="container">
+ <div class="row span12">Copyright © 2014
+ <a href="http://www.apache.org/">The Apache Software Foundation</a>.
+ All Rights Reserved.
+
+ </div>
+
+ <?xml version="1.0" encoding="UTF-8"?>
+<div class="row-fluid">Apache Slider, Slider, Apache, and the Apache Incubator logo are trademarks of The Apache Software Foundation.</div>
+
+ <p id="poweredBy" class="pull-right">
+ <a href="http://maven.apache.org/" title="Built by Maven" class="poweredBy">
+ <img class="builtBy" alt="Built by Maven" src="./images/logos/maven-feather.png" />
+ </a>
+ </p>
+
+ </div>
+ </footer>
+ </body>
+</html>
Propchange: incubator/slider/site/content/dependencies.html
------------------------------------------------------------------------------
svn:eol-style = native