You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2014/07/07 21:03:40 UTC
svn commit: r1608561 [3/16] - in /incubator/slider/site/content: ./ css/
design/ design/registry/ design/specification/ developing/ docs/
docs/configuration/ docs/slider_specs/ downloads/ fonts/ images/ js/
release_notes/
Propchange: incubator/slider/site/content/css/bootstrap.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/css/bootstrap.css.map
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/css/bootstrap.css.map?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/css/bootstrap.css.map (added)
+++ incubator/slider/site/content/css/bootstrap.css.map Mon Jul 7 19:03:38 2014
@@ -0,0 +1 @@
[... 3 lines stripped ...]
Added: incubator/slider/site/content/css/bootstrap.min.css
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/css/bootstrap.min.css?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/css/bootstrap.min.css (added)
+++ incubator/slider/site/content/css/bootstrap.min.css Mon Jul 7 19:03:38 2014
@@ -0,0 +1,7 @@
+/*!
+ * Bootstrap v3.1.1 (http://getbootstrap.com)
+ * Copyright 2011-2014 Twitter, Inc.
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
+ */
+
[... 3 lines stripped ...]
Propchange: incubator/slider/site/content/css/bootstrap.min.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/css/slider.css
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/css/slider.css?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/css/slider.css (added)
+++ incubator/slider/site/content/css/slider.css Mon Jul 7 19:03:38 2014
@@ -0,0 +1,84 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+body {
+ /* for fixed top navbar */
+ padding-top: 70px;
+ font-size: 16px;
+}
+ul.nav li.dropdown:hover > ul.dropdown-menu {
+ /* so nav submenus open on hover */
+ display: block;
+}
+#sidebar {
+ font-size: 14px;
+}
+#sociallinks td {
+ /* no lines in the link table */
+ border-top: none;
+}
+#bannertext {
+ margin-top: 10px;
+ text-align: left;
+}
+.clear {
+ clear: both;
+}
+#content {
+ padding: 0 8px 40px;
+}
+#content h1 {
+ margin-bottom: 0.5em;
+}
+#content h2 {
+ margin-bottom: 0.5em;
+ border-bottom: 1px solid #CCCCCC;
+ padding-bottom: 0.25em;
+}
+#content h3 {
+ margin-bottom: 0.5em;
+}
+#content img {
+ vertical-align: middle;
+}
+#footer {
+ border-top: 1px solid #CCCCCC;
+ color: #666666;
+ font-size: 0.8em;
+ padding: 8px 8px;
+ text-align: center;
+}
+#asf-logo {
+ float: left;
+ padding-top: 15px;
+}
+
+#download-button-sidebar {
+ width: 60%;
+ margin-left: auto;
+ margin-right: auto;
+}
+
+div.copyright {
+ width: 60%;
+ margin-left: auto;
+ margin-right: auto;
+}
+
+code {
+ /* override nowrap in bootstrap */
+ white-space: normal;
+}
Propchange: incubator/slider/site/content/css/slider.css
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/design/architecture.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/design/architecture.html?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/design/architecture.html (added)
+++ incubator/slider/site/content/design/architecture.html Mon Jul 7 19:03:38 2014
@@ -0,0 +1,279 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href="/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/css/bootstrap-theme.min.css" rel="stylesheet">
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <link href="/css/slider.css" rel="stylesheet" type="text/css">
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+ <title></title>
+
+</head>
+
+<body>
+<nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+<div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/index.html">Slider</a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Project <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_index"><a href="/index.html">Home</a></li>
+<li id="nav_downloads"><a href="/downloads">Downloads</a></li>
+<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git"><i class="fa fa-external-link"></i> Source</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0"><i class="fa fa-external-link"></i> License</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Community</li>
+<li id="nav_team"><a href="/team.html">Team</a></li>
+<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Tools</li>
+<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER"><i class="fa fa-external-link"></i> Issues</a></li>
+<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop"><i class="fa fa-external-link"></i> Builds</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/design">
+ Design <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li>
+<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li>
+<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li>
+<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li>
+</ul>
+</li>
+
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/developing">
+ Developing <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li>
+<li id="nav_dev_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li>
+<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li>
+<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li>
+<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Using <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li>
+<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li>
+<li id="nav_use_examples"><a href="/docs/examples.html">Examples</a></li>
+<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li>
+<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li>
+<li id="nav_use_security"><a href="/docs/security.html">Security</a></li>
+<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">App Packaging</a></li>
+<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li>
+<li id="nav_use_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ ASF Links <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li><a href="http://www.apache.org"><i class="fa fa-external-link"></i> Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html"><i class="fa fa-external-link"></i> Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/"><i class="fa fa-external-link"></i> Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html"><i class="fa fa-external-link"></i> Thanks</a></li>
+</ul>
+</li>
+</ul>
+
+ </div>
+
+</div>
+</nav>
+
+
+<div class="container-fluid">
+<div class="row">
+ <div class="col-md-2" id="sidebar">
+ <div style="text-align: center">
+ <h1><a href="/index.html">Apache Slider (incubating)</a></h1>
+ <hr>
+Latest release: <strong>0.30-incubating</strong><br>
+ <br>
+ <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a>
+ </div>
+ <hr>
+ <a href="http://incubator.apache.org"><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a>
+ </div>
+ <div class="col-md-8 col-md-offset-1">
+
+ <div id="content">
+
+ <h1 class="title"></h1>
+
+ <!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h1 id="apache-slider-architecture">Apache Slider Architecture</h1>
+<h2 id="summary">Summary</h2>
+<p>Slider is a YARN application to deploy non-YARN-enabled applications in a YARN cluster</p>
+<p>Slider consists of a YARN application master, the "Slider AM", and a client
+application which communicates with YARN and the Slider AM via remote procedure
+calls and/or REST requests. The client application offers command line access
+ as well as low-level API access for test purposes</p>
+<p>The deployed application must be a program that can be run across a pool of
+YARN-managed servers, dynamically locating its peers. It is not Slider's
+responsibility to configure up the peer servers, apart from some initial
+application-specific application instance configuration. (The full requirements
+of an application are <a href="/docs/slider_specs/application_needs.md">described in another document</a>.</p>
+<p>Every application instance is described as a set of one or more <em>component</em>; each
+component can have a different program/command, and a different set of configuration
+options and parameters.</p>
+<p>The AM takes the details on which roles to start, and requests a YARN container
+for each component; It then monitors the state of the application instance, receiving messages
+from YARN when a remotely executed process finishes. It then deploys another instance of
+that component.</p>
+<h2 id="slider-packaging">Slider Packaging</h2>
+<p>A key goal of Slider is to support the deployment of existing applications into
+a YARN application instance, without having to extend Slider itself. </p>
+<h2 id="am-architecture">AM Architecture</h2>
+<p>The application master consists of</p>
+<ol>
+<li>The AM engine which handles all integration with external services, specifically YARN and any Slider clients</li>
+<li>A <em>provider</em> specific to deploying a class of applications.</li>
+<li>The Application State. </li>
+</ol>
+<p>The Application State is the model of the application instance, containing</p>
+<ol>
+<li>A specification of the desired state of the application instance -the number of instances of each role, their YARN and process memory requirements and some other options. </li>
+<li>A map of the current instances of each role across the YARN cluster, including reliability statistics of each node in the application instance used.</li>
+<li><a href="/design/rolehistory.html">The Role History</a> -a record of which nodes roles were deployed on for re-requesting the same nodes in future. This is persisted to disk and re-read if present, for faster application startup times.</li>
+<li>Queues of track outstanding requests, released and starting nodes</li>
+</ol>
+<p>The Application Engine integrates with the outside world: the YARN Resource Manager ("the RM"), and the node-specific Node Managers, receiving events from the services, requesting or releasing containers via the RM, and starting applications on assigned containers.</p>
+<p>After any notification of a change in the state of the cluster (or an update to the client-supplied cluster specification), the Application Engine passes the information on to the Application State class, which updates its state and then returns a list of cluster operations to be submitted: requests for containers of different types -potentially on specified nodes, or requests to release containers.</p>
+<p>As those requests are met and allocation messages passed to the Application Engine, it works with the Application State to assign them to specific components, then invokes the provider to build up the launch context for that application.</p>
+<p>The provider has the task of populating container requests with the file references, environment variables and commands needed to start the provider's supported programs. </p>
+<p>The core provider deploys a minimal agent on the target containers, then, as the agent checks in to the agent provider's REST API, executes commands issued to it. </p>
+<p>The set of commands this agent executes focuses on downloading archives from HDFS, expanding them, then running Python scripts which perform the
+actual configuration and execution of the target problem -primarily through template expansion.</p>
+<p>To summarize: Slider is not an classic YARN analysis application, which allocates and schedules work across the cluster in short-to-medium life containers with the lifespan of a query or an analytics session, but instead for an application with a lifespan of days to months. Slider works to keep the actual state of its application cluster to match the desired state, while the application has the tasks of recovering from node failure, locating peer nodes and working with data in an HDFS filesystem. </p>
+<p>As such it is one of the first applications designed to use YARN as a platform for long-lived services -Samza being the other key example. These application's needs of YARN are different, and their application manager design is focused around maintaining the distributed application in its desired state rather than the ongoing progress of submitted work.</p>
+<p>The clean model-view-controller split was implemented to isolate the model and aid mock testing of large clusters with simulated scale, and hence increase confidence that Slider can scale to work in large YARN clusters and with larger application instances. </p>
+<h3 id="failure-model">Failure Model</h3>
+<p>The application master is designed to be a <a href="https://www.usenix.org/legacy/events/hotos03/tech/full_papers/candea/candea.pdf">crash-only application</a>, clients are free to terminate
+the application instance by asking YARN directly. </p>
+<p>There is an RPC call to stop the application instance - this is a nicety which includes a message in the termination log, and
+could, in future, perhaps warn the provider that the application instance is being torn down. That is a potentially dangerous feature
+to add -as provider implementors may start to expect the method to be called reliably. Slider is designed to fail without
+warning, to rebuild its state on a YARN-initiated restart, and to be manually terminated without any advance notice.</p>
+<h3 id="rpc-interface">RPC Interface</h3>
+<p>The RPC interface allows the client to query the current application state, and to update it by pushing out a new JSON specification. </p>
+<p>The core operations are</p>
+<ul>
+<li><code>getJSONClusterStatus()</code>: get the status of the application instance as a JSON document.</li>
+<li><code>flexCluster()</code> update the desired count of role instances in the running application instance.</li>
+<li><code>stopCluster</code> stop the application instance</li>
+</ul>
+<p>There are some other low-level operations for extra diagnostics and testing, but they are of limited importancs </p>
+<p>The <code>flexCluster()</code> call takes a JSON application instance specification and forwards it to the AM -which extracts the desired counts of each role to update the Application State. A change in the desired size of the application instance, is treated as any reported failure of node:
+it triggers a re-evaluation of the application state, building up the list of container add and release requests to make of
+the YARN resource manager.</p>
+<p>The final operation, <code>stopCluster()</code>, stops the application instance. </p>
+<h3 id="security-and-identity">Security and Identity</h3>
+<p>Slider's security model is described in detail in <a href="/docs/security.html">an accompanying document</a></p>
+<p>A Slider application instance is expected to access data belonging to the user creating the instance. </p>
+<p>In a secure YARN cluster, this is done by acquiring Kerberos tokens in the client when the application instance is updated, tokens which
+are propagated to the Slider AM and thence to the deployed application containers themselves. These
+tokens are valid for a finite time period. </p>
+<p>HBase has always required keytab files to be installed on every node in the Hadoop for it to have secure access -this requirement
+holds for Slider-deployed HBase clusters. Slider does not itself adopt the responsibility of preparing or distributing these files;
+this must be done via another channel.</p>
+<p>In Hadoop 2.2, the tokens for communication between the Slider AM and YARN expire after -by default- 72 hours. The
+HDFS tokens will also expire after some time period. This places an upper bound on the lifespan of a Slider application (or any
+other long-lived YARN application) in a secure Hadoop cluster. </p>
+<p>In an insecure Hadoopp cluster, the Slider AM and its containers are likely to run in a different OS account from the submitting user.
+To enable access to the database files as that submitting use, the identity of the user is provided when the AM is created; the
+AM will pass this same identity down to the created containers. This information <em>identifies</em> the user -but does not <em>authenticate</em> them: they are trusted to be who they claim to be.</p>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2014 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.<br />
+ Site created with <a href="http://getbootstrap.com/">Bootstrap</a> including icons from <a href="http://glyphicons.com/">GLYPHICONS</a> and <a href="http://fontawesome.io/">Font Awesome</a>.
+ </p>
+ </div>
+ </div>
+ </div>
+
+</div>
+</div>
+
+<script type="text/javascript">
+
+</script>
+</body>
+</html>
Propchange: incubator/slider/site/content/design/architecture.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/design/index.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/design/index.html?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/design/index.html (added)
+++ incubator/slider/site/content/design/index.html Mon Jul 7 19:03:38 2014
@@ -0,0 +1,206 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href="/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/css/bootstrap-theme.min.css" rel="stylesheet">
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <link href="/css/slider.css" rel="stylesheet" type="text/css">
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+ <title></title>
+
+</head>
+
+<body>
+<nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+<div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/index.html">Slider</a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Project <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_index"><a href="/index.html">Home</a></li>
+<li id="nav_downloads"><a href="/downloads">Downloads</a></li>
+<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git"><i class="fa fa-external-link"></i> Source</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0"><i class="fa fa-external-link"></i> License</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Community</li>
+<li id="nav_team"><a href="/team.html">Team</a></li>
+<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Tools</li>
+<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER"><i class="fa fa-external-link"></i> Issues</a></li>
+<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop"><i class="fa fa-external-link"></i> Builds</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/design">
+ Design <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li>
+<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li>
+<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li>
+<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li>
+</ul>
+</li>
+
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/developing">
+ Developing <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li>
+<li id="nav_dev_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li>
+<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li>
+<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li>
+<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Using <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li>
+<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li>
+<li id="nav_use_examples"><a href="/docs/examples.html">Examples</a></li>
+<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li>
+<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li>
+<li id="nav_use_security"><a href="/docs/security.html">Security</a></li>
+<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">App Packaging</a></li>
+<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li>
+<li id="nav_use_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ ASF Links <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li><a href="http://www.apache.org"><i class="fa fa-external-link"></i> Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html"><i class="fa fa-external-link"></i> Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/"><i class="fa fa-external-link"></i> Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html"><i class="fa fa-external-link"></i> Thanks</a></li>
+</ul>
+</li>
+</ul>
+
+ </div>
+
+</div>
+</nav>
+
+
+<div class="container-fluid">
+<div class="row">
+ <div class="col-md-2" id="sidebar">
+ <div style="text-align: center">
+ <h1><a href="/index.html">Apache Slider (incubating)</a></h1>
+ <hr>
+Latest release: <strong>0.30-incubating</strong><br>
+ <br>
+ <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a>
+ </div>
+ <hr>
+ <a href="http://incubator.apache.org"><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a>
+ </div>
+ <div class="col-md-8 col-md-offset-1">
+
+ <div id="content">
+
+ <h1 class="title"></h1>
+
+ <!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h1 id="apache-slider-architecture">Apache Slider Architecture</h1>
+<ul>
+<li><a href="/design/architecture.html">Overview</a></li>
+<li><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li><a href="/design/specification/index.html">Specification</a></li>
+<li><a href="/design/registry/index.html">Service Registry</a></li>
+<li><a href="/design/rolehistory.html">Role history</a> </li>
+</ul>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2014 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.<br />
+ Site created with <a href="http://getbootstrap.com/">Bootstrap</a> including icons from <a href="http://glyphicons.com/">GLYPHICONS</a> and <a href="http://fontawesome.io/">Font Awesome</a>.
+ </p>
+ </div>
+ </div>
+ </div>
+
+</div>
+</div>
+
+<script type="text/javascript">
+
+</script>
+</body>
+</html>
Propchange: incubator/slider/site/content/design/index.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/design/registry/a_YARN_service_registry.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/design/registry/a_YARN_service_registry.html?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/design/registry/a_YARN_service_registry.html (added)
+++ incubator/slider/site/content/design/registry/a_YARN_service_registry.html Mon Jul 7 19:03:38 2014
@@ -0,0 +1,406 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href="/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/css/bootstrap-theme.min.css" rel="stylesheet">
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <link href="/css/slider.css" rel="stylesheet" type="text/css">
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+ <title></title>
+
+</head>
+
+<body>
+<nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+<div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/index.html">Slider</a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Project <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_index"><a href="/index.html">Home</a></li>
+<li id="nav_downloads"><a href="/downloads">Downloads</a></li>
+<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git"><i class="fa fa-external-link"></i> Source</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0"><i class="fa fa-external-link"></i> License</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Community</li>
+<li id="nav_team"><a href="/team.html">Team</a></li>
+<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Tools</li>
+<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER"><i class="fa fa-external-link"></i> Issues</a></li>
+<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop"><i class="fa fa-external-link"></i> Builds</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/design">
+ Design <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li>
+<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li>
+<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li>
+<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li>
+</ul>
+</li>
+
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/developing">
+ Developing <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li>
+<li id="nav_dev_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li>
+<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li>
+<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li>
+<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Using <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li>
+<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li>
+<li id="nav_use_examples"><a href="/docs/examples.html">Examples</a></li>
+<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li>
+<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li>
+<li id="nav_use_security"><a href="/docs/security.html">Security</a></li>
+<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">App Packaging</a></li>
+<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li>
+<li id="nav_use_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ ASF Links <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li><a href="http://www.apache.org"><i class="fa fa-external-link"></i> Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html"><i class="fa fa-external-link"></i> Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/"><i class="fa fa-external-link"></i> Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html"><i class="fa fa-external-link"></i> Thanks</a></li>
+</ul>
+</li>
+</ul>
+
+ </div>
+
+</div>
+</nav>
+
+
+<div class="container-fluid">
+<div class="row">
+ <div class="col-md-2" id="sidebar">
+ <div style="text-align: center">
+ <h1><a href="/index.html">Apache Slider (incubating)</a></h1>
+ <hr>
+Latest release: <strong>0.30-incubating</strong><br>
+ <br>
+ <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a>
+ </div>
+ <hr>
+ <a href="http://incubator.apache.org"><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a>
+ </div>
+ <div class="col-md-8 col-md-offset-1">
+
+ <div id="content">
+
+ <h1 class="title"></h1>
+
+ <!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h1 id="a-yarn-service-registry-for-apache-slider">A YARN Service Registry for Apache Slider</h1>
+<h2 id="april-2014">April 2014</h2>
+<h1 id="introduction">Introduction</h1>
+<p>This document looks at the needs and options of a service registry.</p>
+<p>The core issue is that as the location(s) of a dynamically deployed application are unknown, the standard Hadoop and Java configuration model of some form of text files containing hostnames, ports and URLS no longer works. You cannot define up-front where a service will be.</p>
+<p>Some Hadoop applications -HBase and Accumulo -have solved this with custom ZK bindings. This works for the specific clients, but requires hbase and accumulo client JARs in order to be able to work with the content. (or a re-implementation with knowledge of the non-standard contents of the ZK nodes)</p>
+<p>Other YARN applications will need to publish their bindings - this includes, but is not limited to- Slider deployed applications. Again, these applications can use their own registration and binding model, which would again require custom clients to locate the registry information and parse the contents.</p>
+<p>YARN provides some minimal publishing of AM remote endpoints: a URL to what is assumed to be a Web UI (not a REST API), and an IPC port. The URL is displayed in the YARN UI -in which case it is accessed via a proxy which (currently) only support HTTP GET operations. The YARN API call to list all applications can be used to locate a named instance of an application by (user, application-type, name), and then obtain the raw URL and IPC endpoints. This enumeration process is an O(apps) operation on the YARN RM and only provides access to those two endpoints. Even with the RAW URL, REST operations have proven "troublesome", due to a web filter which redirects all direct requests to the proxy -unless it comes from the same host as the proxy.</p>
+<p>Hadoop client applications tend to retrieve all their configuration information from files in the local filesystem, hadoop-site.xml, hdfs-site.xml, hbase-site.xml, etc. This requires the configuration files to be present on all systems. Tools such as Ambari can keep the files in the server up to date -assuming a low rate of change- ---but these tools do nothing for the client applications themselves. It is up to the cluster clients to (somehow) retrieve these files, and to keep their copies up to date. <em>This is a problem that exists with today's applications</em>. </p>
+<p>As an example, if a YARN client does not know the value of "yarn.application.classpath", it cannot successfully deploy any application in the YARN cluster which needs the cluster-side Hadoop and YARN JARs on its application master's classpath. This is not a theoretical problem, as some clusters have a different classpath from the default: without a correct value the Slider AM does not start. And, as it is designed to be run remotely, it cannot rely on a local installation of YARN to provide the correct values (<a href="https://issues.apache.org/jira/browse/YARN-973">YARN-973</a>).</p>
+<h1 id="what-do-we-need">What do we need?</h1>
+<p><strong>Discovery</strong>: An IPC and URL discovery system for service-aware applications to use to look up a service to which it wishes to talk to. This is not an ORB -it's not doing redirection -, but it is something that needs to be used before starting IPC or REST communications. </p>
+<p><strong>Configuration</strong>: A way for clients of a service to retrieve more configuration data than simply the service endpoints. For example: everything needed to create a site.xml document.</p>
+<h2 id="client-side">Client-side</h2>
+<ul>
+<li>
+<p>Allow clients of a YARN application to locate the service instance and its service ports (web, IPC, REST...) efficiently even on a large YARN cluster. </p>
+</li>
+<li>
+<p>Allow clients to retrieve configuration values which can be processed client-side into the configuration files and options which the application needs</p>
+</li>
+<li>
+<p>Give clients confidence that the service with which they interact is the one they expect to interact with -not another potentially malicious service deployed by a different user. </p>
+</li>
+<li>
+<p>clients to be able to watch a service and retrieve notification of changes</p>
+</li>
+<li>
+<p>cross-language support.</p>
+</li>
+</ul>
+<h2 id="for-all-services">For all Services</h2>
+<ul>
+<li>
+<p>Allow services to publish their binding details for the AM and of code running in the containers (which may be published by the containers)</p>
+</li>
+<li>
+<p>Use entries in registry as a way of enforcing uniqueness of the instance (app, owner, name)? </p>
+</li>
+<li>
+<p>values to update when a service is restarted on a different host</p>
+</li>
+<li>
+<p>values to indicate when a service is not running. This may be implicit "no entry found" or explicit "service exists but not running"</p>
+</li>
+<li>
+<p>Services to be able to act as clients to other services</p>
+</li>
+</ul>
+<h2 id="for-slider-services-and-presumably-others">For Slider Services (and presumably others)</h2>
+<ul>
+<li>
+<p>Ability to publish information about configuration documents that can be retrieved -and URLs</p>
+</li>
+<li>
+<p>Ability to publish facts internal to the application (e.g. agent reporting URLs)</p>
+</li>
+<li>
+<p>Ability to use service paths as a way to ensure a single instance of a named service can be deployed by a user</p>
+</li>
+</ul>
+<h2 id="management-and-paas-uis">Management and PaaS UIs</h2>
+<ul>
+<li>
+<p>Retrieve lists of web UI URLs of AM and of deployed components</p>
+</li>
+<li>
+<p>Enum components and their status</p>
+</li>
+<li>
+<p>retrieve dynamic assignments of IPC ports</p>
+</li>
+<li>
+<p>retrieve dynamic assignments of JMX ports</p>
+</li>
+<li>
+<p>retrieve any health URLs for regular probes</p>
+</li>
+<li>
+<p>Listen to changes in the service mix -the arrival and departure of service instances, as well as changes in their contents.</p>
+</li>
+</ul>
+<h2 id="other-needs">Other Needs</h2>
+<ul>
+<li>
+<p>Registry-configured applications. In-cluster applications should be able to subscribe to part of the registry
+to pick up changes that affect them -both for their own application configuration, and for details about
+applications on which they depend themselves.</p>
+</li>
+<li>
+<p>Knox: get URLs that need to be converted into remote paths</p>
+</li>
+<li>
+<p>Cloud-based deployments: work on virtual infrastructures where hostnames are unpredictable.</p>
+</li>
+</ul>
+<h1 id="open-source-registry-code">Open Source Registry code</h1>
+<p>What can we use to implement this from ASF and ASF-compatible code? </p>
+<h2 id="zookeeper">Zookeeper</h2>
+<p>We'd need a good reason not to use this. There are still some issues</p>
+<ol>
+<li>
+<p>Limits on amount of published data?</p>
+</li>
+<li>
+<p>Load limits, especially during cluster startup, or if a 500-mapper job all wants to do a lookup.</p>
+</li>
+<li>
+<p>Security story</p>
+</li>
+<li>
+<p>Impact of other ZK load on the behaviour of the service registry -will it cause problems if overloaded -and are they recoverable?</p>
+</li>
+</ol>
+<h2 id="apache-curator">Apache Curator</h2>
+<p>Netflix's core curator -now <a href="http://curator.apache.org/">Apache Curator</a>- framework adds a lot to make working with ZK easier, including pluggable retry policies, binding tools and other things.</p>
+<p>There is also its "experimental" <a href="http://curator.apache.org/curator-x-discovery-server/index.html">service discovery framework</a>, which</p>
+<ol>
+<li>
+<p>Allows a service to register a URL with a name and unique ID (and custom metadata). multiple services of a given name can be registered</p>
+</li>
+<li>
+<p>Allows a service to register >1 URL.</p>
+</li>
+<li>
+<p>Has a service client which performs lookup and can cache results.</p>
+</li>
+<li>
+<p>Has a REST API</p>
+</li>
+</ol>
+<p>Limitations</p>
+<ul>
+<li>
+<p>The service discovery web UI and client does not work with the version of
+Jackson (1.8.8) in Hadoop 2.4. The upgraded version in Hadoop 2.5 is compatible <a href="https://issues.apache.org/jira/browse/HADOOP-10104">HADOOP-10104</a>.</p>
+</li>
+<li>
+<p>The per-entry configuration payload attempts to get jason to perform Object/JSON mapping with the classname provided as an attribute in the JSON. This destroys all ability of arbitrary applications to parse the published data, as well as cross-language clients -is brittle and morally wrong from a data-sharing perspective.</p>
+<p>{</p>
+<p>"name" : "name",
+ "id" : "service",
+ "address" : "localhost",
+ "port" : 8080,
+ "sslPort" : 443,
+ "payload" : {
+ "@class" : "org.apache.slider.core.registry.ServiceInstanceData",
+ "externalView" : {
+ "key" : "value"
+ }
+ },
+ "registrationTimeUTC" : 1397249829062,
+ "serviceType" : "DYNAMIC",
+ "uriSpec" : {
+ "parts" : [ {
+ "value" : "http:",
+ "variable" : false
+ }, {
+ "value" : ":",
+ "variable" : false
+ } ]
+ }
+}</p>
+</li>
+</ul>
+<h2 id="helix-service-registry"><a href="http://helix.apache.org/0.7.0-incubating-docs/recipes/service_discovery.html">Helix Service Registry</a></h2>
+<p>This is inside Helix somewhere, used in LI in production at scale -and worth looking at. LI separate their Helix Zookeeper Quorum from their application-layer quorum, to isolate load.</p>
+<p>Notable features</p>
+<ol>
+<li>The registry is also the liveness view of the deployed application. Client's aren't watching the service registry for changes, they are watching Helix's model of the deployed application.</li>
+<li>The deployed application can pick up changes to its state the same way, allowing for live application manipulation.</li>
+<li>Tracks nodes that continually join/leave the group and drops them as unreliable.</li>
+</ol>
+<h2 id="twill-service-registry">Twill Service Registry</h2>
+<p>Twill's <a href="http://twill.incubator.apache.org/apidocs/index.html">service registry code</a>, lets applications register a <a href="http://twill.incubator.apache.org/apidocs/org/apache/twill/discovery/Discoverable.html">(hostname, port)</a> pair in the registry by a name, a name by which clients can look up and enumerate all services with a specific name.</p>
+<p>Clients can subscribe to changes in the list of services with a specific name -so picking up the arrival and departure of instances, and probe to see if a previously discovered entity is still registered.</p>
+<p>Zookeeper- and in-memory registry implementations are provided.</p>
+<p>One nice feature about this architecture -and Twill in general- is that its general single-method callback model means that it segues nicely into Java-8 lambda-expressions. This is something to retain in a YARN-wide service registry.</p>
+<p>Comparing it to curator, it offers a proper subset of curator's registered services <a href="http://curator.apache.org/apidocs/org/apache/curator/x/discovery/ServiceInstance.html">ServiceInstance</a> -implying that you could publish and retrieve Curator-registered services via a new implementation of Twill's DiscoveryService. This would require extensions to the curator service discovery client allow ZK nodes to be watched for changes. This is a feature that would be useful in many use cases -such as watching service availability across a cluster, or simply blocking until a dependent service was launched.</p>
+<p>As with curator, the amount of information that can be published isn't enough for management tools to make effective use of the service registration, while for slider there's no way to publish configuration data. However a YARN registry will inevitably be a superset of the Twill client's enumerated and retrieved data -so if its registration API were sufficient to register a minimal service, supporting the YARN registry via Twill's existing API should be straightforward.</p>
+<h2 id="twitter-commons-service-registration">Twitter Commons Service Registration</h2>
+<p><a href="https://github.com/twitter/commons">Twitter Commons</a> has a service registration library, which allows for registration of sets of servers, <a href="http://twitter.github.io/commons/apidocs/com/twitter/common/service/registration/package-tree.html">publishing the hostname and port of each</a>., along with a map of string properties.</p>
+<p>Zookeeper based, it suffices if all servers are identical and only publishing single (hostname, port) pairs for callers.</p>
+<h2 id="airbnb-smartstack">AirBnB Smartstack</h2>
+<p>SmartStack is <a href="http://nerds.airbnb.com/smartstack-service-discovery-cloud/">Air BnB's cloud-based service discovery system</a>.</p>
+<p>It has two parts, <em>Nerve</em> and <em>Synapse</em>:</p>
+<p><a href="https://github.com/airbnb/nerve"><strong>Nerve</strong></a> is a ruby agent designed to monitor processes and register healthy instances in ZK (or to a mock reporter). It includes <a href="https://github.com/airbnb/nerve/tree/master/lib/nerve/service_watcher">probes for TCP ports, HTTP and rabbitMQ</a>. It's <a href="https://github.com/airbnb/nerve/blob/master/lib/nerve/service_watcher.rb">a fairly simple liveness monitor</a>.</p>
+<p><a href="https://github.com/airbnb/synapse"><strong>Synapse</strong></a> takes the data and uses it to configure <a href="http://haproxy.1wt.eu/">HAProxy instances</a>. HAProxy handles the load balancing, queuing and integrating liveness probes into the queues. Synapse generates all the configuration files for an instance -but also tries to reconfigure the live instances via their socket APIs, </p>
+<p>Alongside these, AirBnB have another published project on Github, <a href="https://github.com/airbnb/optica">Optica</a>, which is a web application for nodes to register themselves with (POST) and for others to query. It publishes events to RabbitMQ, and again uses ZK to store state.</p>
+<p>AirBnB do complain a bit about ZK and its brittleness. They do mention that they suspect it is due to bugs in the Ruby ZK client library. This may be exacerbated by in-cloud deployments. Hard-coding the list of ZK nodes may work for a physical cluster, but in a virtualized cluster, the hostnames/IP Addresses of those nodes may change -leading to a meta-discovery problem: how to find the ZK quorum -especially if you can't control the DNS servers.</p>
+<h2 id="apache-directory"><a href="http://directory.apache.org/apacheds/">Apache Directory</a></h2>
+<p>This is an embeddable LDAP server</p>
+<ul>
+<li>
+<p>Embeddable inside Java apps</p>
+</li>
+<li>
+<p>Supports Kerberos alongside X.500 auth. It can actually act as a Key server and TGT if desired.</p>
+</li>
+<li>
+<p>Supports DNS and DHCP queries.</p>
+</li>
+<li>
+<p>Accessible via classic LDAP APIs.</p>
+</li>
+</ul>
+<p>This isn't a registry service directly, though LDAP queries do make enumeration of services <em>and configuration data</em> straightforward. As LDAP libraries are common across languages -even built in to the Java runtime- LDAP support makes publishing information to arbitrary clients relatively straightforward.</p>
+<p>If service information were to be published via LDAP, then it should allow IT-managed LDAP services to both host this information, and publish configuration data. This would be relevant for classic Hadoop applications if we were to move the Configuration class to support back-end configuration sources beyond XML files on the classpath.</p>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2014 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.<br />
+ Site created with <a href="http://getbootstrap.com/">Bootstrap</a> including icons from <a href="http://glyphicons.com/">GLYPHICONS</a> and <a href="http://fontawesome.io/">Font Awesome</a>.
+ </p>
+ </div>
+ </div>
+ </div>
+
+</div>
+</div>
+
+<script type="text/javascript">
+
+</script>
+</body>
+</html>
Propchange: incubator/slider/site/content/design/registry/a_YARN_service_registry.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/design/registry/index.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/design/registry/index.html?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/design/registry/index.html (added)
+++ incubator/slider/site/content/design/registry/index.html Mon Jul 7 19:03:38 2014
@@ -0,0 +1,222 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href="/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/css/bootstrap-theme.min.css" rel="stylesheet">
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <link href="/css/slider.css" rel="stylesheet" type="text/css">
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+ <title></title>
+
+</head>
+
+<body>
+<nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+<div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/index.html">Slider</a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Project <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_index"><a href="/index.html">Home</a></li>
+<li id="nav_downloads"><a href="/downloads">Downloads</a></li>
+<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git"><i class="fa fa-external-link"></i> Source</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0"><i class="fa fa-external-link"></i> License</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Community</li>
+<li id="nav_team"><a href="/team.html">Team</a></li>
+<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Tools</li>
+<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER"><i class="fa fa-external-link"></i> Issues</a></li>
+<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop"><i class="fa fa-external-link"></i> Builds</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/design">
+ Design <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li>
+<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li>
+<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li>
+<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li>
+</ul>
+</li>
+
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/developing">
+ Developing <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li>
+<li id="nav_dev_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li>
+<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li>
+<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li>
+<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Using <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li>
+<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li>
+<li id="nav_use_examples"><a href="/docs/examples.html">Examples</a></li>
+<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li>
+<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li>
+<li id="nav_use_security"><a href="/docs/security.html">Security</a></li>
+<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">App Packaging</a></li>
+<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li>
+<li id="nav_use_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ ASF Links <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li><a href="http://www.apache.org"><i class="fa fa-external-link"></i> Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html"><i class="fa fa-external-link"></i> Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/"><i class="fa fa-external-link"></i> Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html"><i class="fa fa-external-link"></i> Thanks</a></li>
+</ul>
+</li>
+</ul>
+
+ </div>
+
+</div>
+</nav>
+
+
+<div class="container-fluid">
+<div class="row">
+ <div class="col-md-2" id="sidebar">
+ <div style="text-align: center">
+ <h1><a href="/index.html">Apache Slider (incubating)</a></h1>
+ <hr>
+Latest release: <strong>0.30-incubating</strong><br>
+ <br>
+ <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a>
+ </div>
+ <hr>
+ <a href="http://incubator.apache.org"><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a>
+ </div>
+ <div class="col-md-8 col-md-offset-1">
+
+ <div id="content">
+
+ <h1 class="title"></h1>
+
+ <!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h1 id="apache-slider-service-registry">Apache Slider Service Registry</h1>
+<p>The service registry is a core part of the Slider Architecture -it is how
+dynamically generated configurations are published for clients to pick up.</p>
+<p>The need for a service registry goes beyond Slider, however. We effectively
+have application-specific registries for HBase and Accumulo, and explicit
+registries in Apache Helix and Apache Twill, as well as re-usable registry
+code in Apache Curator.</p>
+<p><a href="https://issues.apache.org/jira/browse/YARN-913">YARN-913</a> covers the need
+for YARN itself to have a service registry. This would be the ideal ultimate
+solution -it would operate at a fixed location/ZK path, and would be guaranteed
+to be on all YARN clusters, so code could be written expecting it to be there.</p>
+<p>It could also be used to publish binding data from static applications,
+including HBase, Accumulo, Oozie, -applications deployed by management tools.
+Unless/until these applications self-published their binding data, it would
+be the duty of the management tools to do the registration.</p>
+<h2 id="contents">Contents</h2>
+<ol>
+<li><a href="the_YARN_application_registration_and_binding_problem.html">YARN Application Registration and Binding: the Problem</a></li>
+<li><a href="a_YARN_service_registry.html">A YARN Service Registry</a></li>
+<li><a href="initial_registry_design.html">April 2014 Initial Registry Design</a></li>
+<li><a href="service_registry_end_to_end_scenario.html">Service Registry End-to-End Scenarios</a></li>
+<li><a href="p2p_service_registries.html">P2P Service Registries</a></li>
+<li><a href="references.html">References</a></li>
+</ol>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2014 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.<br />
+ Site created with <a href="http://getbootstrap.com/">Bootstrap</a> including icons from <a href="http://glyphicons.com/">GLYPHICONS</a> and <a href="http://fontawesome.io/">Font Awesome</a>.
+ </p>
+ </div>
+ </div>
+ </div>
+
+</div>
+</div>
+
+<script type="text/javascript">
+
+</script>
+</body>
+</html>
Propchange: incubator/slider/site/content/design/registry/index.html
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/slider/site/content/design/registry/initial_registry_design.html
URL: http://svn.apache.org/viewvc/incubator/slider/site/content/design/registry/initial_registry_design.html?rev=1608561&view=auto
==============================================================================
--- incubator/slider/site/content/design/registry/initial_registry_design.html (added)
+++ incubator/slider/site/content/design/registry/initial_registry_design.html Mon Jul 7 19:03:38 2014
@@ -0,0 +1,296 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE- 2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <link href="/css/bootstrap.min.css" rel="stylesheet">
+ <link href="/css/bootstrap-theme.min.css" rel="stylesheet">
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+ <script src="/js/bootstrap.min.js"></script>
+ <link href="/css/slider.css" rel="stylesheet" type="text/css">
+ <link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+ <title></title>
+
+</head>
+
+<body>
+<nav class="navbar navbar-default navbar-fixed-top" role="navigation">
+<div class="container-fluid">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a class="navbar-brand" href="/index.html">Slider</a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Project <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_index"><a href="/index.html">Home</a></li>
+<li id="nav_downloads"><a href="/downloads">Downloads</a></li>
+<li id="nav_source"><a href="https://git-wip-us.apache.org/repos/asf/incubator-slider.git"><i class="fa fa-external-link"></i> Source</a></li>
+<li><a href="http://www.apache.org/licenses/LICENSE-2.0"><i class="fa fa-external-link"></i> License</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Community</li>
+<li id="nav_team"><a href="/team.html">Team</a></li>
+<li id="nav_mailinglists"><a href="/mailing_lists.html">Mailing Lists</a></li>
+<li class="divider"></li>
+<li class="dropdown-header">Tools</li>
+<li id="nav_jira"><a href="https://issues.apache.org/jira/browse/SLIDER"><i class="fa fa-external-link"></i> Issues</a></li>
+<li id="nav_builds"><a href="https://builds.apache.org/job/Slider-develop"><i class="fa fa-external-link"></i> Builds</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/design">
+ Design <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_design_overview"><a href="/design/architecture.html">Overview</a></li>
+<li id="nav_app_needs"><a href="/docs/slider_specs/application_needs.html">Application Needs</a></li>
+<li id="nav_spec"><a href="/design/specification/index.html">Expected Behavior</a></li>
+<li id="nav_registry"><a href="/design/registry/index.html">Service Registry</a></li>
+<li id="nav_role_history"><a href="/design/rolehistory.html">Role History</a></li>
+</ul>
+</li>
+
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="/developing">
+ Developing <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_dev_building"><a href="/developing/building.html">Building</a></li>
+<li id="nav_dev_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_dev_testing"><a href="/developing/testing.html">Testing</a></li>
+<li id="nav_dev_funtesting"><a href="/developing/functional_tests.html">Functional Testing</a></li>
+<li id="nav_dev_manual_testing"><a href="/developing/manual_testing.html">Manual Testing</a></li>
+<li id="nav_releasing"><a href="/developing/releasing.html">Releasing</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ Using <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li id="nav_use_getting_started"><a href="/docs/getting_started.html">Getting Started</a></li>
+<li id="nav_use_manpage"><a href="/docs/manpage.html">Man Page</a></li>
+<li id="nav_use_examples"><a href="/docs/examples.html">Examples</a></li>
+<li id="nav_use_client_config"><a href="/docs/client-configuration.html">Client Configuration</a></li>
+<li id="nav_use_exitcodes"><a href="/docs/exitcodes.html">Client Exit Codes</a></li>
+<li id="nav_use_security"><a href="/docs/security.html">Security</a></li>
+<li id="nav_use_app_package"><a href="/docs/slider_specs/index.html">App Packaging</a></li>
+<li id="nav_use_app_configuration"><a href="/docs/configuration/index.html">App Configuration Model</a></li>
+<li id="nav_use_debugging"><a href="/docs/debugging.html">Debugging</a></li>
+<li id="nav_use_troubleshooting"><a href="/docs/troubleshooting.html">Troubleshooting</a></li>
+</ul>
+</li>
+
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">
+ ASF Links <span class="caret"></span>
+ </a>
+<ul class="dropdown-menu">
+<li><a href="http://www.apache.org"><i class="fa fa-external-link"></i> Apache Software Foundation</a></li>
+<li><a href="http://www.apache.org/foundation/sponsorship.html"><i class="fa fa-external-link"></i> Sponsorship</a></li>
+<li><a href="http://www.apache.org/security/"><i class="fa fa-external-link"></i> Security</a></li>
+<li><a href="http://www.apache.org/foundation/thanks.html"><i class="fa fa-external-link"></i> Thanks</a></li>
+</ul>
+</li>
+</ul>
+
+ </div>
+
+</div>
+</nav>
+
+
+<div class="container-fluid">
+<div class="row">
+ <div class="col-md-2" id="sidebar">
+ <div style="text-align: center">
+ <h1><a href="/index.html">Apache Slider (incubating)</a></h1>
+ <hr>
+Latest release: <strong>0.30-incubating</strong><br>
+ <br>
+ <a id="download-button-sidebar" class="btn btn-success btn-block" href="/downloads/" role="button">Download</a>
+ </div>
+ <hr>
+ <a href="http://incubator.apache.org"><img id="logo" alt="Apache Incubator ™" class="img-responsive" src="http://incubator.apache.org/images/apache-incubator-logo.png"/></a>
+ </div>
+ <div class="col-md-8 col-md-offset-1">
+
+ <div id="content">
+
+ <h1 class="title"></h1>
+
+ <!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h1 id="april-2014-initial-registry-design-for-apache-slider">April 2014 Initial Registry Design for Apache Slider</h1>
+<p>This is the plan for the initial registry design.</p>
+<ol>
+<li>
+<p>Use Apache Curator <a href="http://curator.apache.org/curator-x-discovery/index.html">service discovery code</a>. </p>
+</li>
+<li>
+<p>AMs to register as (user, name). Maybe "service type" if we add that as an option in the slider configs</p>
+</li>
+<li>
+<p>Lift "external view" term from Helix -concept that this is the public view, not internal.</p>
+</li>
+<li>
+<p>application/properties section to list app-wide values</p>
+</li>
+<li>
+<p>application/services section to list public service URLs; publish each as unique-ID-> (human name, URL, human text). code can resolve from UniqueID; UIs can use human data.</p>
+</li>
+<li>
+<p>String Template 2 templates for generation of output (rationale: library for Python Java, .NET)</p>
+</li>
+<li>
+<p>Java CLI to retrieve values from ZK and apply named template (local, hdfs). Include ability to restrict to list of named properties (pattern match).</p>
+</li>
+<li>
+<p>AM to serve up curator service (later -host in RM? elsewhere?)</p>
+</li>
+</ol>
+<h3 id="forwards-compatilibity">forwards-compatilibity</h3>
+<ol>
+<li>
+<p>This initial design will hide the fact that Apache Curator is being used to discover services,
+by storing information in the payload, <code>ServiceInstanceData</code> rather than in (the minimal) curator
+service entries themselves. If we move to an alternate registry, provided we
+can use the same datatype -or map to it- changes should not be visible.</p>
+</li>
+<li>
+<p>The first implementation will not support watching for changes.</p>
+</li>
+</ol>
+<h3 id="initial-templates">Initial templates</h3>
+<ul>
+<li>
+<p>hadoop XML conf files</p>
+</li>
+<li>
+<p>Java properties file</p>
+</li>
+<li>
+<p>HTML listing of services</p>
+</li>
+</ul>
+<h2 id="example-curator-service-entry">Example Curator Service Entry</h2>
+<p>This is the prototype's content</p>
+<p>Toplevel</p>
+<pre class="codehilite"><code>service CuratorServiceInstance{name='slider', id='stevel.test_registry_am', address='192.168.1.101', port=62552, sslPort=null, payload=org.apache.slider.core.registry.info.ServiceInstanceData@4e9af21b, registrationTimeUTC=1397574073203, serviceType=DYNAMIC, uriSpec=org.apache.curator.x.discovery.UriSpec@ef8dacf0}</code></pre>
+
+
+<p>Slider payload.</p>
+<pre class="codehilite"><code>payload=
+{
+ "internalView" : {
+ "endpoints" : {
+ "/agents" : {
+ "value" : "http://stevel-8.local:62552/ws/v1/slider/agents",
+ "protocol" : "http",
+ "type" : "url",
+ "description" : "Agent API"
+ }
+ },
+ "settings" : { }
+ },
+
+ "externalView" : {
+ "endpoints" : {
+ "/mgmt" : {
+ "value" : "http://stevel-8.local:62552/ws/v1/slider/mgmt",
+ "protocol" : "http",
+ "type" : "url",
+ "description" : "Management API"
+ },
+
+ "slider/IPC" : {
+ "value" : "stevel-8.local/192.168.1.101:62550",
+ "protocol" : "org.apache.hadoop.ipc.Protobuf",
+ "type" : "address",
+ "description" : "Slider AM RPC"
+ },
+ "registry" : {
+ "value" : "http://stevel-8.local:62552/ws/registry",
+ "protocol" : "http",
+ "type" : "url",
+ "description" : "Registry"
+ }
+ },
+ "settings" : { }
+ }
+}</code></pre>
+ </div>
+
+ <div id="footer">
+ <a alt="Apache Software Foundation" href="http://www.apache.org">
+ <img id="asf-logo" alt="Apache Software Foundation" src="/images/feather-small.gif" width="100">
+ </a>
+ <div class="copyright">
+ <p>
+ Copyright © 2011-2014 The Apache Software Foundation, Licensed under
+ the <a href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.
+ Apache Slider, Slider, Apache, the Apache feather logo, and the Apache Incubator
+ project logo are trademarks of the <a href="http://www.apache.org">Apache Software Foundation</a>.<br />
+ Site created with <a href="http://getbootstrap.com/">Bootstrap</a> including icons from <a href="http://glyphicons.com/">GLYPHICONS</a> and <a href="http://fontawesome.io/">Font Awesome</a>.
+ </p>
+ </div>
+ </div>
+ </div>
+
+</div>
+</div>
+
+<script type="text/javascript">
+
+</script>
+</body>
+</html>
Propchange: incubator/slider/site/content/design/registry/initial_registry_design.html
------------------------------------------------------------------------------
svn:eol-style = native