You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bu...@apache.org on 2014/07/22 20:17:30 UTC
svn commit: r917125 [11/18] - in /websites/staging/slider/trunk/content: ./
css/ design/ design/registry/ design/specification/ developing/ docs/
docs/configuration/ docs/slider_specs/ downloads/ fonts/ images/ js/
release_notes/
Added: websites/staging/slider/trunk/content/docs/configuration/redesign.html
==============================================================================
--- websites/staging/slider/trunk/content/docs/configuration/redesign.html (added)
+++ websites/staging/slider/trunk/content/docs/configuration/redesign.html Tue Jul 22 18:17:12 2014
@@ -0,0 +1,624 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href="/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/css/bootstrap-theme.min.css" rel="stylesheet">
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <link href="/css/slider.css" rel="stylesheet" type="text/css">
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+ <title></title>
+
+</head>
+
+<body>
+<nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+<div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/index.html">Slider</a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Project <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_index"><a href="/index.html">Home</a></li>
+<li id="nav_downloads"><a href="/downloads">Downloads</a></li>
+<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git"><i class="fa fa-external-link"></i> Source</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0"><i class="fa fa-external-link"></i> License</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Community</li>
+<li id="nav_team"><a href="/team.html">Team</a></li>
+<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Tools</li>
+<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER"><i class="fa fa-external-link"></i> Issues</a></li>
+<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop"><i class="fa fa-external-link"></i> Builds</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/design">
+ Design <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li>
+<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li>
+<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li>
+<li id="nav_ssl_impl"><a href="/design/ssl_implementation.html">SSL Implementation</a></li>
+<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li>
+</ul>
+</li>
+
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/developing">
+ Developing <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li>
+<li id="nav_dev_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li>
+<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li>
+<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li>
+<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Using <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li>
+<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li>
+<li id="nav_use_examples"><a href="/docs/examples.html">Examples</a></li>
+<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li>
+<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li>
+<li id="nav_use_security"><a href="/docs/security.html">Security</a></li>
+<li id="nav_am_agent_ssl"><a href="/docs/ssl.html">Agent to AM SSL</a></li>
+<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">Creating App Packages</a></li>
+<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li>
+<li id="nav_use_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ ASF Links <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li><a href="http://www.apache.org"><i class="fa fa-external-link"></i> Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html"><i class="fa fa-external-link"></i> Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/"><i class="fa fa-external-link"></i> Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html"><i class="fa fa-external-link"></i> Thanks</a></li>
+</ul>
+</li>
+</ul>
+
+ </div>
+
+</div>
+</nav>
+
+
+<div class="container-fluid">
+<div class="row">
+ <div class="col-md-2" id="sidebar">
+ <div style="text-align: center">
+ <h1><a href="/index.html">Apache Slider (incubating)</a></h1>
+ <hr>
+Latest release: <strong>0.40-incubating</strong><br>
+ <br>
+ <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a>
+ </div>
+ <hr>
+ <a href="http://incubator.apache.org"><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a>
+ </div>
+ <div class="col-md-8 col-md-offset-1">
+
+ <div id="content">
+
+ <h1 class="title"></h1>
+
+ <!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h1 id="apache-slider-cluster-specification">Apache Slider Cluster Specification</h1>
+<h3 id="notation">Notation:</h3>
+<p>In this document, a full path to a value is represented as a path
+<code>options/zookeeper.port</code> ; an assigment as <code>options/zookeeper.port=2181</code>.</p>
+<p>A wildcard indicates all entries matching a path: <code>options/zookeeper.*</code>
+or <code>/roles/*/yarn.memory</code></p>
+<h2 id="history">History</h2>
+<p>The Slider cluster specification was implicitly defined in the file
+<code>org.apache.slider.api.ClusterDescription</code>. It had a number of roles</p>
+<ol>
+<li>Persistent representaton of cluster state</li>
+<li>Internal model of desired cluster state within the Application Master.</li>
+<li>Dynamic representation of current cluster state when the AM
+was queried, marshalled over the network as JSON.</li>
+<li>Description of updated state when reconfiguring a running cluster.</li>
+</ol>
+<p>Initially the dynamic status included a complete history of all containers
+-this soon highlit some restrictions on the maximum size of a JSON-formatted
+string in Hadoop's "classic" RPC: 32K, after which the string was silently
+truncated. Accordingly, this history was dropped.</p>
+<p>Having moved to Protocol Buffers as the IPC wire format, with a web view
+alongside, this history could be reconsidered.</p>
+<p>The initial design place most values into the root entry, and relied
+on Jaxon introspection to set and retrieve the values -it was a
+Java-first specification, with no external specificatin or regression tests.</p>
+<p>As the number of entries in the root increased, the design switched to storing
+more attributes into specific sections <em>under</em> the root path:</p>
+<ul>
+<li><code>info</code>: read-only information about the cluster.</li>
+<li><code>statistics</code>: Numeric statistics about the cluster</li>
+</ul>
+<h1 id="sections">Sections</h1>
+<h2 id="root">Root</h2>
+<p>Contains various string and integer values</p>
+<div class="codehilite"><pre>"<span class="n">version</span>"<span class="p">:</span> "1<span class="p">.</span>0"<span class="p">,</span>
+"<span class="n">name</span>"<span class="p">:</span> "<span class="n">test_cluster_lifecycle</span>"<span class="p">,</span>
+"<span class="n">type</span>"<span class="p">:</span> "<span class="n">hbase</span>"<span class="p">,</span>
+"<span class="n">state</span>"<span class="p">:</span> 3<span class="p">,</span>
+"<span class="n">createTime</span>"<span class="p">:</span> 1393512091276<span class="p">,</span>
+"<span class="n">updateTime</span>"<span class="p">:</span> 1393512117286<span class="p">,</span>
+"<span class="n">originConfigurationPath</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">snapshot</span>"<span class="p">,</span>
+"<span class="n">generatedConfigurationPath</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">generated</span>"<span class="p">,</span>
+"<span class="n">dataPath</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">database</span>"<span class="p">,</span>
+</pre></div>
+
+
+<ul>
+<li>
+<p><code>version</code>: version of the JSON file. Not currently used
+to validate version compatibility; at this point in time
+releases may not be able to read existing .json files.</p>
+</li>
+<li>
+<p><code>name</code>: cluster name</p>
+</li>
+<li><code>type</code>: reference to the provider type -this triggers a Hadoop configuration
+property lookup to find the implementation classes.</li>
+<li>
+<p><code>state</code>: an enumeration value of the cluster state.</p>
+<div class="codehilite"><pre><span class="n">int</span> <span class="n">STATE_INCOMPLETE</span> <span class="p">=</span> 0<span class="p">;</span>
+<span class="n">int</span> <span class="n">STATE_SUBMITTED</span> <span class="p">=</span> 1<span class="p">;</span>
+<span class="n">int</span> <span class="n">STATE_CREATED</span> <span class="p">=</span> 2<span class="p">;</span>
+<span class="n">int</span> <span class="n">STATE_LIVE</span> <span class="p">=</span> 3<span class="p">;</span>
+<span class="n">int</span> <span class="n">STATE_STOPPED</span> <span class="p">=</span> 4<span class="p">;</span>
+<span class="n">int</span> <span class="n">STATE_DESTROYED</span> <span class="p">=</span> 5<span class="p">;</span>
+</pre></div>
+
+
+</li>
+</ul>
+<p>Only two states are persisted, "incomplete" and "created", though more
+ are used internally.
+ The <code>incomplete</code> state is used during cluster create/build,
+ allowing an incomplete JSON file to be written
+ -so minimising the window for race conditions on cluster construction.</p>
+<ul>
+<li><code>createTime</code> and <code>updateTime</code>: timestamps, informative only.
+ The <code>createTime</code> value is duplicated in <code>/info/createTimeMillis</code></li>
+<li><code>originConfigurationPath</code>, <code>generatedConfigurationPath</code>, <code>dataPath</code> paths
+used internally -if changed the cluster may not start.</li>
+</ul>
+<p><em>Proposed</em>:
+1. Move all state bar <code>name</code> and cluster state
+into a section <code>/slider-internal</code>.
+1. The cluster state is moved from an enum to a simple
+ boolean, <code>valid</code>, set to true when the cluster JSON
+ has been fully constructed.</p>
+<h2 id="info"><code>/info</code></h2>
+<p>Read-only list of information about the application. Generally this is
+intended to be used for debugging and testing.</p>
+<h3 id="persisted-values-static-information-about-the-file-history">Persisted values: static information about the file history</h3>
+<div class="codehilite"><pre>"<span class="n">info</span>" <span class="p">:</span> <span class="p">{</span>
+ "<span class="n">create</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">deployed</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="p">(</span><span class="n">detached</span> <span class="n">from</span> <span class="n">release</span><span class="o">-</span>2<span class="p">.</span>3<span class="p">.</span>0<span class="p">)</span> <span class="p">@</span><span class="n">dfe46336fbc6a044bc124392ec06b85</span>"<span class="p">,</span>
+ "<span class="n">create</span><span class="p">.</span><span class="n">application</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="n">Slider</span> <span class="n">Core</span><span class="o">-</span>0<span class="p">.</span>13<span class="p">.</span>0<span class="o">-</span><span class="n">SNAPSHOT</span> <span class="n">Built</span> <span class="n">against</span> <span class="n">commit</span># 1<span class="n">a94ee4aa1</span> <span class="n">on</span> <span class="n">Java</span> 1<span class="p">.</span>7<span class="p">.</span>0<span class="n">_45</span> <span class="n">by</span> <span class="n">stevel</span>"<span class="p">,</span>
+ "<span class="n">create</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "2<span class="p">.</span>3<span class="p">.</span>0"<span class="p">,</span>
+ "<span class="n">create</span><span class="p">.</span><span class="n">time</span><span class="p">.</span><span class="n">millis</span>" <span class="p">:</span> "1393512091276"<span class="p">,</span>
+<span class="p">},</span>
+</pre></div>
+
+
+<p><em>Proposed</em>: move persisted info K-V pairs to a section <code>/diagnostics</code>.</p>
+<h3 id="dynamic-values">Dynamic values:</h3>
+<p>whether the AM supports service restart without killing all the containers hosting
+ the role instances:</p>
+<div class="codehilite"><pre>"<span class="n">slider</span><span class="p">.</span><span class="n">am</span><span class="p">.</span><span class="n">restart</span><span class="p">.</span><span class="n">supported</span>" <span class="p">:</span> "<span class="n">false</span>"<span class="p">,</span>
+</pre></div>
+
+
+<p>timestamps of the cluster going live, and when the status query was made</p>
+<div class="codehilite"><pre>"<span class="n">live</span><span class="p">.</span><span class="n">time</span>" <span class="p">:</span> "27 <span class="n">Feb</span> 2014 14<span class="p">:</span>41<span class="p">:</span>56 <span class="n">GMT</span>"<span class="p">,</span>
+"<span class="n">live</span><span class="p">.</span><span class="n">time</span><span class="p">.</span><span class="n">millis</span>" <span class="p">:</span> "1393512116881"<span class="p">,</span>
+"<span class="n">status</span><span class="p">.</span><span class="n">time</span>" <span class="p">:</span> "27 <span class="n">Feb</span> 2014 14<span class="p">:</span>42<span class="p">:</span>08 <span class="n">GMT</span>"<span class="p">,</span>
+"<span class="n">status</span><span class="p">.</span><span class="n">time</span><span class="p">.</span><span class="n">millis</span>" <span class="p">:</span> "1393512128726"<span class="p">,</span>
+</pre></div>
+
+
+<p>yarn data provided to the AM</p>
+<div class="codehilite"><pre>"<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>" <span class="p">:</span> "32"<span class="p">,</span>
+"<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>" <span class="p">:</span> "2048"<span class="p">,</span>
+</pre></div>
+
+
+<p>information about the application and hadoop versions in use. Here
+ the application was built using Hadoop 2.3.0, but is running against the version
+ of Hadoop built for HDP-2.</p>
+<div class="codehilite"><pre>"<span class="n">status</span><span class="p">.</span><span class="n">application</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="n">Slider</span> <span class="n">Core</span><span class="o">-</span>0<span class="p">.</span>13<span class="p">.</span>0<span class="o">-</span><span class="n">SNAPSHOT</span> <span class="n">Built</span> <span class="n">against</span> <span class="n">commit</span># 1<span class="n">a94ee4aa1</span> <span class="n">on</span> <span class="n">Java</span> 1<span class="p">.</span>7<span class="p">.</span>0<span class="n">_45</span> <span class="n">by</span> <span class="n">stevel</span>"<span class="p">,</span>
+"<span class="n">status</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">build</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "2<span class="p">.</span>3<span class="p">.</span>0"<span class="p">,</span>
+"<span class="n">status</span><span class="p">.</span><span class="n">hadoop</span><span class="p">.</span><span class="n">deployed</span><span class="p">.</span><span class="n">info</span>" <span class="p">:</span> "<span class="n">bigwheel</span><span class="o">-</span><span class="n">m16</span><span class="o">-</span>2<span class="p">.</span>2<span class="p">.</span>0 <span class="p">@</span>704<span class="n">f1e463ebc4fb89353011407e965</span>"
+</pre></div>
+
+
+<p>## <code>instances</code></p>
+<p>Information about the live containers in a cluster</p>
+<div class="codehilite"><pre> "<span class="n">instances</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">slider</span>"<span class="p">:</span> <span class="p">[</span> "<span class="n">container_1393511571284_0002_01_000001</span>" <span class="p">],</span>
+ "<span class="n">master</span>"<span class="p">:</span> <span class="p">[</span> "<span class="n">container_1393511571284_0002_01_000003</span>" <span class="p">],</span>
+ "<span class="n">worker</span>"<span class="p">:</span> <span class="p">[</span>
+ "<span class="n">container_1393511571284_0002_01_000002</span>"<span class="p">,</span>
+ "<span class="n">container_1393511571284_0002_01_000004</span>"
+ <span class="p">]</span>
+ <span class="p">},</span>
+</pre></div>
+
+
+<p>There's no information about location, nor is there any history about containers
+that are no longer part of the cluster (i.e. failed & released containers). </p>
+<p>It could be possible to include a list of previous containers,
+though Slider would need to be selective about how many to store
+(or how much detail to retain) on those previous containers.</p>
+<p>Perhaps the list could be allowed to grow without limit, but detail
+only preserved on the last 100. If more containers fail than that,
+there is likely to be a problem which the most recent containers
+will also display.</p>
+<p><em>Proposed</em> </p>
+<ol>
+<li>Return to the full serialization of container state -but only for running containers.</li>
+<li>Have a list of failed containers, but only include last 8; make it a rolling
+buffer. This avoids a significantly failing role to overload the status document.</li>
+</ol>
+<p>## <code>statistics</code></p>
+<p>Statistics on each role. </p>
+<p>They can be divided into counters that only increase</p>
+<div class="codehilite"><pre>"<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+"<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+"<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+"<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+"<span class="n">containers</span><span class="p">.</span><span class="n">requested</span>"<span class="p">:</span> 0
+</pre></div>
+
+
+<p>and those that vary depending upon the current state</p>
+<div class="codehilite"><pre>"<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 0<span class="p">,</span>
+"<span class="n">containers</span><span class="p">.</span><span class="n">active</span><span class="p">.</span><span class="n">requests</span>"<span class="p">:</span> 0<span class="p">,</span>
+"<span class="n">containers</span><span class="p">.</span><span class="n">desired</span>"<span class="p">:</span> 0<span class="p">,</span>
+</pre></div>
+
+
+<ul>
+<li>
+<p>Propose: move these values out of statistics into some other section, as they
+are state, not statistics*</p>
+<p>"statistics": {
+ "worker": {
+ "containers.start.completed": 0,
+ "containers.live": 2,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 2,
+ "containers.requested": 0
+ },
+ "slider": {
+ "containers.unknown.completed": 0,
+ "containers.start.completed": 3,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.surplus": 0
+ },
+ "master": {
+ "containers.start.completed": 0,
+ "containers.live": 1,
+ "containers.start.failed": 0,
+ "containers.active.requests": 0,
+ "containers.failed": 0,
+ "containers.completed": 0,
+ "containers.desired": 1,
+ "containers.requested": 0
+ }
+ },</p>
+</li>
+</ul>
+<p>The <code>/statistics/slider</code> section is unusual in that it provides the aggregate statistics
+of the cluster -this is not obvious. A different name could be used -but
+again, there's a risk of clash with or confusion with a role. </p>
+<p>Better to have a specific <code>/statistics/cluster</code> element,
+and to move the roles' statistics under <code>/statistics/roles</code>:</p>
+<div class="codehilite"><pre>"<span class="n">statistics</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">cluster</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">unknown</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 3<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 1<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">surplus</span>"<span class="p">:</span> 0
+
+ <span class="p">},</span>
+ "<span class="n">roles</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">worker</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 2<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">active</span><span class="p">.</span><span class="n">requests</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">desired</span>"<span class="p">:</span> 2<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">requested</span>"<span class="p">:</span> 0
+ <span class="p">},</span>
+ "<span class="n">master</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">live</span>"<span class="p">:</span> 1<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">start</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">active</span><span class="p">.</span><span class="n">requests</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">failed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">completed</span>"<span class="p">:</span> 0<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">desired</span>"<span class="p">:</span> 1<span class="p">,</span>
+ "<span class="n">containers</span><span class="p">.</span><span class="n">requested</span>"<span class="p">:</span> 0
+ <span class="p">}</span>
+ <span class="p">}</span>
+<span class="p">},</span>
+</pre></div>
+
+
+<p>This approach allows extra statistics sections to be added (perhaps
+by providers), without any changes to the toplevel section.</p>
+<h2 id="options">Options</h2>
+<p>A list of options used by Slider and its providers to build up the AM
+and the configurations of the deployed service components</p>
+<div class="codehilite"><pre>"<span class="n">options</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">zookeeper</span><span class="p">.</span><span class="n">port</span>"<span class="p">:</span> "2181"<span class="p">,</span>
+ "<span class="n">site</span><span class="p">.</span><span class="n">hbase</span><span class="p">.</span><span class="n">master</span><span class="p">.</span><span class="n">startup</span><span class="p">.</span><span class="n">retainassign</span>"<span class="p">:</span> "<span class="n">true</span>"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">cluster</span><span class="p">.</span><span class="n">application</span><span class="p">.</span><span class="n">image</span><span class="p">.</span><span class="n">path</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">hbase</span><span class="p">.</span><span class="n">tar</span><span class="p">.</span><span class="n">gz</span>"<span class="p">,</span>
+ "<span class="n">site</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">defaultFS</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">threshold</span>"<span class="p">:</span> "5"<span class="p">,</span>
+ "<span class="n">site</span><span class="p">.</span><span class="n">fs</span><span class="p">.</span><span class="n">default</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">cluster</span><span class="p">.</span><span class="n">directory</span><span class="p">.</span><span class="n">permissions</span>"<span class="p">:</span> "0770"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">am</span><span class="p">.</span><span class="n">monitoring</span><span class="p">.</span><span class="n">enabled</span>"<span class="p">:</span> "<span class="n">false</span>"<span class="p">,</span>
+ "<span class="n">zookeeper</span><span class="p">.</span><span class="n">path</span>"<span class="p">:</span> "<span class="o">/</span><span class="n">yarnapps_slider_stevel_test_cluster_lifecycle</span>"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">tmp</span><span class="p">.</span><span class="n">dir</span>"<span class="p">:</span> "<span class="n">hdfs</span><span class="p">:</span><span class="o">//</span><span class="n">sandbox</span><span class="p">:</span>8020<span class="o">/</span><span class="n">user</span><span class="o">/</span><span class="n">stevel</span><span class="o">/</span><span class="p">.</span><span class="n">slider</span><span class="o">/</span><span class="n">cluster</span><span class="o">/</span><span class="n">test_cluster_lifecycle</span><span class="o">/</span><span class="n">tmp</span><span class="o">/</span><span class="n">am</span>"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">data</span><span class="p">.</span><span class="n">directory</span><span class="p">.</span><span class="n">permissions</span>"<span class="p">:</span> "0770"<span class="p">,</span>
+ "<span class="n">zookeeper</span><span class="p">.</span><span class="n">hosts</span>"<span class="p">:</span> "<span class="n">sandbox</span>"<span class="p">,</span>
+ "<span class="n">slider</span><span class="p">.</span><span class="n">container</span><span class="p">.</span><span class="n">failure</span><span class="p">.</span><span class="n">shortlife</span>"<span class="p">:</span> "60"
+<span class="p">},</span>
+</pre></div>
+
+
+<p>Some for these options have been created by slider itself ("slider.tmp.dir")
+for internal use -and are cluster specific. If/when the ability to use
+an existing json file as a template for a new cluster is added, having these
+options in the configuration will create problems</p>
+<h1 id="proposed-changes">Proposed Changes</h1>
+<h2 id="move-slider-internal-state-to-slider-internal">Move Slider internal state to <code>/slider-internal</code></h2>
+<p>Move all slider "private" data to an internal section,<code>/slider-internal</code>
+including those in the toplevel directory and in <code>/options</code></p>
+<h2 id="allow-options-and-roles-options-entries-to-take-the-value-null">Allow <code>/options</code> and <code>roles/*/</code> options entries to take the value "null".</h2>
+<p>This would be a definition that the value must be defined before the cluster
+can start. Provider templates could declare this.</p>
+<h2 id="make-client-configuration-retrieval-hierarchical-and-maybe-move-out-of-the">Make client configuration retrieval hierarchical -and maybe move out of the</h2>
+<p>status</p>
+<p>The current design assumes that it is a -site.xml file being served up. This
+does not work for alternate file formats generated by the Provider.</p>
+<h2 id="role-options">Role Options</h2>
+<p>The <code>/roles/$ROLENAME/</code> clauses each provide options for a
+specific role.</p>
+<p>This includes
+1. <code>role.instances</code>: defines the number of instances of a role to create
+1. <code>env.</code> environment variables for launching the container
+1. <code>yarn.</code> properties to configure YARN requests.
+1. <code>jvm.heapsize</code>: an option supported by some providers to
+fix the heap size of a component.
+1. <code>app.infoport</code>: an option supported by some providers (e.g. HBase)
+to fix the port to which a role (master or worker) binds its web UI.</p>
+<div class="codehilite"><pre> "<span class="n">worker</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "768"<span class="p">,</span>
+ "<span class="n">env</span><span class="p">.</span><span class="n">MALLOC_ARENA_MAX</span>"<span class="p">:</span> "4"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">worker</span>"<span class="p">,</span>
+ "<span class="n">jvm</span><span class="p">.</span><span class="n">heapsize</span>"<span class="p">:</span> "512<span class="n">M</span>"<span class="p">,</span>
+ "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span>
+ "<span class="n">app</span><span class="p">.</span><span class="n">infoport</span>"<span class="p">:</span> "0"
+ <span class="p">},</span>
+</pre></div>
+
+
+<p>In a live cluster, the role information also includes status information
+about the cluster.</p>
+<div class="codehilite"><pre> "<span class="n">master</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "1024"<span class="p">,</span>
+ "<span class="n">env</span><span class="p">.</span><span class="n">MALLOC_ARENA_MAX</span>"<span class="p">:</span> "4"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">requested</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">master</span>"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">failed</span><span class="p">.</span><span class="n">starting</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">actual</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">jvm</span><span class="p">.</span><span class="n">heapsize</span>"<span class="p">:</span> "512<span class="n">M</span>"<span class="p">,</span>
+ "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">releasing</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">failed</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "0"<span class="p">,</span>
+ "<span class="n">app</span><span class="p">.</span><span class="n">infoport</span>"<span class="p">:</span> "0"
+ <span class="p">}</span>
+</pre></div>
+
+
+<p>The role <code>slider</code> represents the Slider Application Master itself.</p>
+<div class="codehilite"><pre> "<span class="n">slider</span>"<span class="p">:</span> <span class="p">{</span>
+ "<span class="n">yarn</span><span class="p">.</span><span class="n">memory</span>"<span class="p">:</span> "256"<span class="p">,</span>
+ "<span class="n">env</span><span class="p">.</span><span class="n">MALLOC_ARENA_MAX</span>"<span class="p">:</span> "4"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">instances</span>"<span class="p">:</span> "1"<span class="p">,</span>
+ "<span class="n">role</span><span class="p">.</span><span class="n">name</span>"<span class="p">:</span> "<span class="n">slider</span>"<span class="p">,</span>
+ "<span class="n">jvm</span><span class="p">.</span><span class="n">heapsize</span>"<span class="p">:</span> "256<span class="n">M</span>"<span class="p">,</span>
+ "<span class="n">yarn</span><span class="p">.</span><span class="n">vcores</span>"<span class="p">:</span> "1"<span class="p">,</span>
+ <span class="p">},</span>
+</pre></div>
+
+
+<h3 id="proposed">Proposed:</h3>
+<ol>
+<li>move all dynamic role status to its own clauses.</li>
+<li>use a simple inheritance model from <code>/options</code></li>
+<li>don't allow role entries to alter the cluster state. </li>
+</ol>
+<h3 id="proposed-clientproperties-continues-return-key-val-pairs">Proposed: <code>/clientProperties</code> continues return Key-val pairs</h3>
+<p>The <code>/clientProperties</code> section will remain, with key-val pairs of type
+string, the expectation being this is where providers can insert specific
+single attributes for client applications.</p>
+<p>These values can be converted to application-specific files on the client,
+in code -as done today in the Slider CLI-, or via template expansion (beyond
+the scope of this document.</p>
+<h3 id="proposed-alongside-clientproperties-comes-clientfiles">Proposed: alongside <code>/clientProperties</code> comes <code>/clientfiles</code></h3>
+<p>This section will list all files that an application instance can generate
+for clients, along with with a description.</p>
+<div class="codehilite"><pre>"<span class="o">/</span><span class="n">clientfiles</span><span class="o">/</span><span class="n">hbase</span><span class="o">-</span><span class="n">site</span><span class="p">.</span><span class="n">xml</span>"<span class="p">:</span> "<span class="n">site</span> <span class="n">information</span> <span class="k">for</span> <span class="n">HBase</span>"
+"<span class="o">/</span><span class="n">clientfiles</span><span class="o">/</span><span class="n">log4</span><span class="p">.</span><span class="k">properties</span>"<span class="p">:</span> "<span class="n">log4</span><span class="p">.</span><span class="n">property</span> <span class="n">file</span>"
+</pre></div>
+
+
+<p>A new CLI command would be added to retrieve a client file.
+1. The specific file must be named.
+1. If it is not present, an error must be raised.
+1. If it is present, it is downloaded and output to the console/to a named
+destination file/directory <code>--outfile <file></code> and <code>--outdir <dir></code>
+1. If the <code>--list</code> argument is provided, the list of available files is
+returned (e.g.) </p>
+<div class="codehilite"><pre><span class="n">hbase</span><span class="o">-</span><span class="n">site</span><span class="p">.</span><span class="n">xml</span><span class="p">:</span> <span class="n">site</span> <span class="n">information</span> <span class="k">for</span> <span class="n">HBase</span>
+<span class="n">log4</span><span class="p">.</span><span class="k">properties</span><span class="p">:</span> <span class="n">log4</span><span class="p">.</span><span class="n">property</span> <span class="n">file</span>
+</pre></div>
+
+
+<p><em>No attempt to parse/process the body of the messages will be returned.</em></p>
+<p>In a REST implementation of the client API, /clientconf would be a path
+to the list of options; each file a path underneath.</p>
+<p>Client configuration file retrieval outside the status completely;
+the status just lists the possible values; a separate call returns them.</p>
+<p>This will permit binary content to be retrieved, and avoid any marshalling
+problems and inefficiencies.</p>
+<p>With this change, there will now be two ways to generate client configuration
+files</p>
+<ul>
+<li>Client-side: as today</li>
+<li>Server-side: via the provider</li>
+</ul>
+<p>Client side is more extensible as it allows for arbitrary clients; server-side
+is restricted to those files which the application provider is capable of
+generating. The advantage of the server-side option is that for those files
+about which the provider is aware of, they will be visible through the
+REST and Web UIs, so trivially retrieved.</p>
+<h3 id="stop-intermixing-role-specification-with-role-current-state">Stop intermixing role specification with role current state</h3>
+<p>Create a new section, <code>rolestatus</code>, which lists the current status
+of the roles: how many are running vs requested, how many are being
+released.</p>
+<p>There's some overlap here with the <code>/statistics</code> field, so we should
+either merge them or clearly separate the two. Only the <code>role.failed</code>
+properties match entries in the statistics -perhaps they should be cut.</p>
+<h4 id="provider-specific-status">provider-specific status</h4>
+<p>Allow providers to publish information to the status, in their
+own section.</p>
+<p>There already is support for providers updating the cluster status
+in Slider 12.1 and earlier, but it has flaws</p>
+<p>A key one is that it is done sychronously on a <code>getStatus()</code> call;
+as providers may perform a live query of their status (example, the HBase
+provider looks up the Web UI ports published by HBase to zookeeper),
+there's overhead, and if the operation blocks (example: when HBase hasn't
+ever been deployed and the zookeeper path is empty), then the status
+call blocks.</p>
+<p><em>Proposed:</em></p>
+<ol>
+<li>There is a specific <code>/provider</code> section</li>
+<li>There's no restriction on what JSON is permitted in this section.</li>
+<li>Providers may make their own updates to the application state to read and
+write this block -operations that are asynchronous to any status queries.</li>
+</ol>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2014 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.<br />
+ Site created with <a href="http://getbootstrap.com/">Bootstrap</a> including icons from <a href="http://glyphicons.com/">GLYPHICONS</a> and <a href="http://fontawesome.io/">Font Awesome</a>.
+ </p>
+ </div>
+ </div>
+ </div>
+
+</div>
+</div>
+
+<script type="text/javascript">
+
+</script>
+</body>
+</html>
Added: websites/staging/slider/trunk/content/docs/configuration/resolved-resources.json
==============================================================================
--- websites/staging/slider/trunk/content/docs/configuration/resolved-resources.json (added)
+++ websites/staging/slider/trunk/content/docs/configuration/resolved-resources.json Tue Jul 22 18:17:12 2014
@@ -0,0 +1,22 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "metadata": {
+ "description": "example of a resources file"
+ },
+
+ "global": {
+ "yarn.vcores": "1",
+ "yarn.memory": "512"
+ },
+
+ "components": {
+ "master": {
+ "instances": "1",
+ "yarn.memory": "1024"
+ },
+ "worker": {
+ "instances":"5"
+ }
+ }
+}
\ No newline at end of file