You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by xi...@apache.org on 2016/10/19 17:21:45 UTC
svn commit: r1765686 [40/42] - in /samza/site: ./ archive/ community/
contribute/ img/0.11/ img/0.11/learn/ img/0.11/learn/documentation/
img/0.11/learn/documentation/comparisons/
img/0.11/learn/documentation/container/ img/0.11/learn/documentation/int...
Added: samza/site/learn/documentation/0.11/yarn/application-master.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/yarn/application-master.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/yarn/application-master.html (added)
+++ samza/site/learn/documentation/0.11/yarn/application-master.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,231 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Application Master</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/documentation/latest/yarn/application-master.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Application Master</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>YARN is Hadoop’s next-generation cluster manager. It allows developers to deploy and execute arbitrary commands on a grid. If you’re unfamiliar with YARN, or the concept of an ApplicationMaster (AM), please read Hadoop’s <a href="http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html">YARN</a> page.</p>
+
+<h3 id="integration">Integration</h3>
+
+<p>Samza’s main integration with YARN comes in the form of a Samza ApplicationMaster. This is the chunk of code responsible for managing a Samza job in a YARN grid. It decides what to do when a stream processor fails, which machines a Samza job’s <a href="../container/samza-container.html">containers</a> should run on, and so on.</p>
+
+<p>When the Samza ApplicationMaster starts up, it does the following:</p>
+
+<ol>
+<li>Creates the <a href="../container/coordinator-stream.html#JobCoordinator">Job Coordinator</a> which bootstraps the Job Model and config from the <a href="../container/coordinator-stream.html">Coordinator Stream</a>.</li>
+<li>Starts a JMX server on a random port.</li>
+<li>Instantiates a metrics registry and reporters to keep track of relevant metrics.</li>
+<li>Registers the AM with YARN’s RM.</li>
+<li>Gets the total number of partitions for the Samza job using each input stream’s PartitionManager (see the <a href="../container/streams.html">Streams</a> page for details).</li>
+<li>Read the total number of containers requested from the Samza job’s configuration.</li>
+<li>Assign each partition to a container (called a Task Group in Samza’s AM dashboard).</li>
+<li>Make a <a href="http://hadoop.apache.org/docs/current/api/org/apache/hadoop/yarn/api/records/ResourceRequest.html">ResourceRequest</a> to YARN for each container. If <a href="yarn-host-affinity.html">host-affinity</a> is enabled on the job, the AM uses the container locality information provided by the Job Coordinator and requests for the same host in the ResourceRequest.</li>
+<li>Starts a ContainerAllocator thread that matches allocated containers and starts the container process.</li>
+<li>Poll the YARN RM every second to check for allocated and released containers.</li>
+</ol>
+
+<p>From this point on, the ApplicationMaster just reacts to events from the RM and delegates it to the ContainerAllocator thread.</p>
+
+<h3 id="fault-tolerance">Fault Tolerance</h3>
+
+<p>Whenever a container is allocated, the AM will work with the YARN NM to start a SamzaContainer (with appropriate partitions assigned to it) in the container. If a container fails with a non-zero return code, the AM will request a new container, and restart the SamzaContainer. If a SamzaContainer fails too many times, too quickly, the ApplicationMaster will fail the whole Samza job with a non-zero return code. See the yarn.container.retry.count and yarn.container.retry.window.ms <a href="../jobs/configuration.html">configuration</a> parameters for details.</p>
+
+<p>When the AM receives a reboot signal from YARN, it will throw a SamzaException. This will trigger a clean and successful shutdown of the AM (YARN won’t think the AM failed).</p>
+
+<p>If the AM, itself, fails, YARN will handle restarting the AM. When the AM is restarted, all containers that were running will be killed, and the AM will start from scratch. The same list of operations, shown above, will be executed. The AM will request new containers for its SamzaContainers, and proceed as though it has just started for the first time. YARN has a yarn.resourcemanager.am.max-retries configuration parameter that’s defined in <a href="http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml">yarn-site.xml</a>. This configuration defaults to 1, which means that, by default, a single AM failure will cause your Samza job to stop running.</p>
+
+<h3 id="security">Security</h3>
+
+<p>The Samza dashboard’s HTTP access is currently un-secured, even when using YARN in secure-mode. This means that users with access to a YARN grid could port-scan a Samza ApplicationMaster’s HTTP server, and open the dashboard in a browser to view its contents. Sensitive configuration can be viewed by anyone, in this way, and care should be taken. There are plans to secure Samza’s ApplicationMaster using <a href="http://docs.hortonworks.com/HDPDocuments/HDP1/HDP-1.3.0/bk_installing_manually_book/content/rpm-chap14-2-3-1.html">Hadoop’s security</a> features (<a href="http://en.wikipedia.org/wiki/SPNEGO">SPENAGO</a>).</p>
+
+<p>See Samza’s <a href="../operations/security.html">security</a> page for more details.</p>
+
+<h2 id="isolation-»"><a href="isolation.html">Isolation »</a></h2>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/documentation/latest/yarn/application-master.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/documentation/0.11/yarn/isolation.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/yarn/isolation.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/yarn/isolation.html (added)
+++ samza/site/learn/documentation/0.11/yarn/isolation.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,218 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Isolation</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/documentation/latest/yarn/isolation.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Isolation</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>When running Samza jobs in a shared, distributed environment, the stream processors can have an impact on one another’s performance. A stream processor that uses 100% of a machine’s CPU will slow down all other stream processors on the machine.</p>
+
+<p>One of <a href="http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html">YARN</a>’s responsibilities is to manage resources so that this doesn’t happen. Each of YARN’s Node Managers (NM) has a chunk of “resources” dedicated to it. The YARN Resource Manager (RM) will only allow a container to be allocated on a NM if it has enough resources to satisfy the container’s needs.</p>
+
+<p>YARN currently supports resource management for memory and CPU.</p>
+
+<h3 id="memory">Memory</h3>
+
+<p>YARN will automatically enforce memory limits for all containers that it executes. All containers must have a max-memory size defined when they’re created. If the sum of all memory usage for processes associated with a single YARN container exceeds this maximum, YARN will kill the container.</p>
+
+<p>Samza supports memory limits using the yarn.container.memory.mb and yarn.am.container.memory.mb configuration parameters. Keep in mind that this is simply the amount of memory YARN will allow a <a href="../container/samza-container.html">SamzaContainer</a> or <a href="application-master.html">ApplicationMaster</a> to have. You’ll still need to configure your heap settings appropriately using task.opts, when using Java (the default is -Xmx160M). See the <a href="../jobs/configuration.html">Configuration</a> and <a href="../jobs/packaging.html">Packaging</a> pages for details.</p>
+
+<h3 id="cpu">CPU</h3>
+
+<p>YARN has the concept of a virtual core. Each NM is assigned a total number of virtual cores (32, by default). When a container request is made, it must specify how many virtual cores it needs. The YARN RM will only assign the container to a NM that has enough virtual cores to satisfy the request.</p>
+
+<h4 id="cgroups">CGroups</h4>
+
+<p>Unlike memory, which YARN can enforce itself (by looking at the /proc folder), YARN can’t enforce CPU isolation, since this must be done at the Linux kernel level. One of YARN’s interesting new features is its support for Linux <a href="https://www.kernel.org/doc/Documentation/cgroup-v1/cgroups.txt">CGroups</a>. CGroups are a way to control process utilization at the kernel level in Linux.</p>
+
+<p>If YARN is setup to use CGroups, then YARN will guarantee that a container will get at least the amount of CPU that it requires. Currently, YARN will give you more CPU, if it’s available. For details on enforcing “at most” CPU usage, see <a href="https://issues.apache.org/jira/browse/YARN-810">YARN-810</a>.</p>
+
+<p>See <a href="http://riccomini.name/posts/hadoop/2013-06-14-yarn-with-cgroups/">this blog post</a> for details on setting up YARN with CGroups.</p>
+
+<h2 id="host-affinity-&-yarn-»"><a href="../yarn/yarn-host-affinity.html">Host Affinity & YARN »</a></h2>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/documentation/latest/yarn/isolation.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/documentation/0.11/yarn/yarn-host-affinity.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/yarn/yarn-host-affinity.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/yarn/yarn-host-affinity.html (added)
+++ samza/site/learn/documentation/0.11/yarn/yarn-host-affinity.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,296 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Host Affinity & YARN</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/documentation/latest/yarn/yarn-host-affinity.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Host Affinity & YARN</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>In Samza, containers are the units of physical parallelism that runs on a set of machines. Each container is essentially a process that runs one or more stream tasks. Each task instance consumes one or more partitions of the input streams and is associated with its own durable data store.</p>
+
+<p>We define a <em>Stateful Samza Job</em> as the Samza job that uses a key-value store in its implementation, along with an associated changelog stream. In stateful samza jobs, a task may be configured to use multiple stores. For each store there is a 1:1 mapping between the task instance and the data store. Since the allocation of containers to machines in the Yarn cluster is completely left to Yarn, Samza does not guarantee that a container (and hence, its associated task(s)) gets deployed on the same machine. Containers can get shuffled in any of the following cases:</p>
+
+<ol>
+<li>When a job is upgraded by pointing <code>yarn.package.path</code> to the new package path and re-submitted.</li>
+<li>When a job is simply restarted by Yarn or the user</li>
+<li>When a container failure or premption triggers the SamzaAppMaster to re-allocate on another available resource</li>
+</ol>
+
+<p>In any of the above cases, the task’s co-located data needs to be restored every time a container starts-up. Restoring data each time can be expensive, especially for applications that have a large data set. This behavior slows the start-up time for the job so much that the job is no longer “near realtime”. Furthermore, if multiple stateful samza jobs restart around the same time in the cluster and they all share the same changelog system, then it is possible to quickly saturate the changelog system’s network and cause a DDoS.</p>
+
+<p>For instance, consider a Samza job performing a Stream-Table join. Typically, such a job requires the dataset to be available on all processors before they begin processing the input stream. The dataset is usually large (order > 1TB) read-only data that will be used to join or add attributes to incoming messages. The job may initialize this cache by populating it with data directly from a remote store or changelog stream. This cache initialization happens each time the container is restarted. This causes significant latency during job start-up.</p>
+
+<p>The solution, then, is to simply persist the state store on the machine in which the container process is executing and re-allocate the same host for the container each time the job is restarted, in order to re-use the persisted state. Thus, the ability of Samza to allocate a container to the same machine across job restarts is referred to as <strong><em>host-affinity</em></strong>. Samza leverages host-affinity to enhance our support for local state re-use.</p>
+
+<h2 id="how-does-it-work?">How does it work?</h2>
+
+<p>When a stateful Samza job is deployed in Yarn, the state stores for the tasks are co-located in the current working directory of Yarn’s application attempt.</p>
+
+<div class="highlight"><pre><code class="bash"><span class="nv">container_working_dir</span><span class="o">=</span><span class="k">${</span><span class="nv">yarn</span><span class="p">.nodemanager.local-dirs</span><span class="k">}</span>/usercache/<span class="k">${</span><span class="nv">user</span><span class="k">}</span>/appcache/application_<span class="k">${</span><span class="nv">appid</span><span class="k">}</span>/container_<span class="k">${</span><span class="nv">contid</span><span class="k">}</span>/
+
+<span class="c"># Data Stores</span>
+ls <span class="k">${</span><span class="nv">container_working_dir</span><span class="k">}</span>/state/<span class="k">${</span><span class="nv">store</span><span class="p">-name</span><span class="k">}</span>/<span class="k">${</span><span class="nv">task_name</span><span class="k">}</span>/</code></pre></div>
+
+<p>This allows the Node Manager’s (NM) DeletionService to clean-up the working directory once the application completes or fails. In order to re-use local state store, the state store needs to be persisted outside the scope of NM’s deletion service. The cluster administrator should set this location as an environment variable in Yarn - <code>LOGGED_STORE_BASE_DIR</code>.</p>
+
+<p><img src="/img/0.11/learn/documentation/yarn/samza-host-affinity.png" alt="samza-host-affinity"></p>
+
+<p>Each time a task commits, Samza writes the last materialized offset from the changelog stream to the checksumed file on disk. This is also done on container shutdown. Thus, there is an <em>OFFSET</em> file associated with each state stores' changelog partitions, that is consumed by the tasks in the container.</p>
+
+<div class="highlight"><pre><code class="bash"><span class="k">${</span><span class="nv">LOGGED_STORE_BASE_DIR</span><span class="k">}</span>/<span class="k">${</span><span class="nv">job</span><span class="p">.name</span><span class="k">}</span>-<span class="k">${</span><span class="nv">job</span><span class="p">.id</span><span class="k">}</span>/<span class="k">${</span><span class="nv">store</span><span class="p">.name</span><span class="k">}</span>/<span class="k">${</span><span class="nv">task</span><span class="p">.name</span><span class="k">}</span>/OFFSET</code></pre></div>
+
+<p>Now, when a container restarts on the same machine after the OFFSET file exists, the Samza container:</p>
+
+<ol>
+<li>Opens the persisted store on disk</li>
+<li>Reads the OFFSET file</li>
+<li>Restores the state store from the OFFSET value</li>
+</ol>
+
+<p>This significantly reduces the state restoration time on container start-up as we no longer consume from the beginning of the changelog stream. If the OFFSET file doesn’t exist, it creates the state store and consumes from the oldest offset in the changelog to re-create the state. Since the OFFSET file is written on each commit after flushing the store, the recorded offset is guaranteed to correspond to the current contents of the store or some older point, but never newer. This gives at least once semantics for state restore. Therefore, the changelog entries must be idempotent.</p>
+
+<p>It is necessary to periodically clean-up unused or orphaned state stores on the machines to manage disk-space. This feature is being worked on in <a href="https://issues.apache.org/jira/browse/SAMZA-656">SAMZA-656</a>.</p>
+
+<p>In order to re-use local state, Samza has to sucessfully claim the specific hosts from the Resource Manager (RM). To support this, the Samza containers write their locality information to the <a href="../container/coordinator-stream.html">Coordinator Stream</a> every time they start-up successfully. Now, the Samza Application Master (AM) can identify the last known host of a container via the <a href="../container/coordinator-stream.html">Job Coordinator</a>(JC) and the application is no longer agnostic of the container locality. On a container failure (due to any of the above cited reasons), the AM includes the hostname of the expected resource in the <a href="https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java#L239%5D">ResourceRequest</a>.</p>
+
+<p>Note that the Yarn cluster has to be configured to use <a href="https://hadoop.apache.org/docs/r2.6.0/hadoop-yarn/hadoop-yarn-site/FairScheduler.html">Fair Scheduler</a> with continuous-scheduling enabled. With continuous scheduling, the scheduler continuously iterates through all nodes in the cluster, instead of relying on the nodes' heartbeat, and schedules work based on previously known status for each node, before relaxing locality. Hence, the scheduler takes care of relaxing locality after the configured delay. This approach can be considered as a “<em>best-effort stickiness</em>” policy because it is possible that the requested node is not running or does not have sufficient resources at the time of request (even though the state in the data stores may be persisted). For more details on the choice of Fair Scheduler, please refer the <a href="https://issues.apache.org/jira/secure/attachment/12726945/DESIGN-SAMZA-617-2.pdf">design doc</a>.</p>
+
+<h2 id="configuring-yarn-cluster-to-support-host-affinity">Configuring YARN cluster to support Host Affinity</h2>
+
+<ol>
+<li>Enable local state re-use by setting the <code>LOGGED_STORE_BASE_DIR</code> environment variable in yarn-env.sh
+<div class="highlight"><pre><code class="bash"><span class="nb">export </span><span class="nv">LOGGED<em>STORE</em>BASE_DIR</span><span class="o">=</span><path-for-state-stores></code></pre></div>
+Without this configuration, the state stores are not persisted upon a container shutdown. This will effectively mean you will not re-use local state and hence, host-affinity becomes a moot operation.</li>
+<li><p>Configure Yarn to use Fair Scheduler and enable continuous-scheduling in yarn-site.xml
+<div class="highlight"><pre><code class="xml"><span class="nt"><property></span>
+<span class="nt"><name></span>yarn.resourcemanager.scheduler.class<span class="nt"></name></span>
+<span class="nt"><description></span>The class to use as the resource scheduler.<span class="nt"></description></span>
+<span class="nt"><value></span>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler<span class="nt"></value></span>
+<span class="nt"></property></span>
+<span class="nt"><property></span>
+<span class="nt"><name></span>yarn.scheduler.fair.continuous-scheduling-enabled<span class="nt"></name></span>
+<span class="nt"><description></span>Enable Continuous Scheduling of Resource Requests<span class="nt"></description></span>
+<span class="nt"><value></span>true<span class="nt"></value></span>
+<span class="nt"></property></span>
+<span class="nt"><property></span>
+<span class="nt"><name></span>yarn.scheduler.fair.locality-delay-node-ms<span class="nt"></name></span>
+<span class="nt"><description></span>Delay time in milliseconds before relaxing locality at node-level<span class="nt"></description></span>
+<span class="nt"><value></span>1000<span class="nt"></value></span> <span class="c"><!-- Should be tuned per requirement --></span>
+<span class="nt"></property></span>
+<span class="nt"><property></span>
+<span class="nt"><name></span>yarn.scheduler.fair.locality-delay-rack-ms<span class="nt"></name></span>
+<span class="nt"><description></span>Delay time in milliseconds before relaxing locality at rack-level<span class="nt"></description></span>
+<span class="nt"><value></span>1000<span class="nt"></value></span> <span class="c"><!-- Should be tuned per requirement --></span>
+<span class="nt"></property></span></code></pre></div></p></li>
+<li><p>Configure Yarn Node Manager SIGTERM to SIGKILL timeout to be reasonable time s.t. Node Manager will give Samza Container enough time to perform a clean shutdown in yarn-site.xml
+<div class="highlight"><pre><code class="xml"><span class="nt"><property></span>
+<span class="nt"><name></span>yarn.nodemanager.sleep-delay-before-sigkill.ms<span class="nt"></name></span>
+<span class="nt"><description></span>No. of ms to wait between sending a SIGTERM and SIGKILL to a container<span class="nt"></description></span>
+<span class="nt"><value></span>600000<span class="nt"></value></span> <span class="c"><!-- Set it to 10min to allow enough time for clean shutdown of containers --></span>
+<span class="nt"></property></span></code></pre></div></p></li>
+<li><p>The Yarn <a href="https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/RackAwareness.html">Rack Awareness</a> feature is not required and does not change the behavior of Samza Host Affinity. However, if Rack Awareness is configured in the cluster, make sure the DNSToSwitchMapping implementation is robust. Any failures could cause container requests to fall back to the defaultRack. This will cause ContainerRequests to not match the preferred host, which will degrade Host Affinity. For details, see <a href="https://issues.apache.org/jira/browse/SAMZA-886">SAMZA-866</a></p></li>
+</ol>
+
+<h2 id="configuring-a-samza-job-to-use-host-affinity">Configuring a Samza job to use Host Affinity</h2>
+
+<p>Any stateful Samza job can leverage this feature to reduce the Mean Time To Restore (MTTR) of its state stores by setting <code>yarn.samza.host-affinity.enabled</code> to true.</p>
+
+<div class="highlight"><pre><code class="bash">yarn.samza.host-affinity.enabled<span class="o">=</span><span class="nb">true</span> <span class="c"># Default: false</span></code></pre></div>
+
+<p>Enabling this feature for a stateless Samza job should not have any adverse effect on the job.</p>
+
+<h2 id="host-affinity-guarantees">Host-affinity Guarantees</h2>
+
+<p>As you have observed, host-affinity cannot be guaranteed all the time due to varibale load distribution in the Yarn cluster. Hence, this is a best-effort policy that Samza provides. However, certain scenarios are worth calling out where these guarantees may be hard to achieve or are not applicable.</p>
+
+<ol>
+<li><em>When the number of containers and/or container-task assignment changes across successive application runs</em> - We may be able to re-use local state for a subset of partitions. Currently, there is no logic in the Job Coordinator to handle partitioning of tasks among containers intelligently. Handling this is more involved as relates to <a href="https://issues.apache.org/jira/browse/SAMZA-336">auto-scaling</a> of the containers. However, with <a href="https://issues.apache.org/jira/browse/SAMZA-906">task-container mapping</a>, this will work better for typical container count adjustments.</li>
+<li><em>When SystemStreamPartitionGrouper changes across successive application runs</em> - When the grouper logic used to distribute the partitions across containers changes, the data in the Coordinator Stream (for changelog-task partition assignment etc) and the data stores becomes invalid. Thus, to be safe, we should flush out all state-related data from the Coordinator Stream. An alternative is to overwrite the Task-ChangelogPartition assignment message and the Container Locality message in the Coordinator Stream, before starting up the job again.</li>
+</ol>
+
+<h2 id="writing-to-hdfs-»"><a href="../hdfs/producer.html">Writing to HDFS »</a></h2>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/documentation/latest/yarn/yarn-host-affinity.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/documentation/0.11/yarn/yarn-security.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/yarn/yarn-security.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/yarn/yarn-security.html (added)
+++ samza/site/learn/documentation/0.11/yarn/yarn-security.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,91 @@
+<hr>
+
+<p>layout: page</p>
+
+<h2 id="title:-yarn-security">title: YARN Security</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>You can run a Samza job on a secure YARN cluster. YARN uses Kerberos as its authentication and authorization mechanism. See <a href="https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-common/SecureMode.html">this article</a> for details on Hadoop in secure mode.</p>
+
+<h3 id="delegation-token-management-strategy">Delegation token management strategy</h3>
+
+<p>One of the challenges for long-lived application running on a secure YARN cluster is its token renewal strategy. Samza takes the following approach to manage token creation and renewal.</p>
+
+<ol>
+<li><p>Client running Samza app needs to kinit into KDC with his credentials and add the HDFS delegation tokens to the launcher context before submitting the application.</p></li>
+<li><p>Next, client prepares the local resources for the application as follows.
+2.1. First, it creates a staging directory on HDFS. This directory is only accessible by the running user and used to store resources required for Application Master (AM) and Containers.
+2.2. Client then adds the keytab as a local resource in the container launcher context for AM.
+2.3. Finally, it sends the corresponding principal and the path to the keytab file in the staging directory to the coordinator stream. Samza currently uses the staging directory to store both the keytab and refreshed tokens because the access to the directory is secured via Kerberos.</p></li>
+<li><p>Once the resource is allocated for the Application Master, the Node Manager will localizes app resources from HDFS using the HDFS delegation tokens in the launcher context. Same rule applies to Container localization too. </p></li>
+<li><p>When Application Master starts, it localizes the keytab file into its working directory and reads the principal from the coordinator stream.</p></li>
+<li><p>The Application Master periodically re-authenticate itself with the given principal and keytab. In each iteration, it creates new delegation tokens and stores them in the given job specific staging directory on HDFS.</p></li>
+<li><p>Each running container will get new delegation tokens from the credentials file on HDFS before the current ones expire.</p></li>
+<li><p>Application Master and Containers don’t communicate with each other for that matter. Each side proceeds independently by reading or writing the tokens on HDFS.</p></li>
+</ol>
+
+<p>By default, any HDFS delegation token has a maximum life of 7 days (configured by <code>dfs.namenode.delegation.token.max-lifetime</code> in hdfs-site.xml) and the token is normally renewed every 24 hours (configured by <code>dfs.namenode.delegation.token.renew-interval</code> in hdfs-site.xml). What if the Application Master dies and needs restarts after 7 days? The original HDFS delegation token stored in the launcher context will be invalid no matter what. Luckily, Samza can rely on Resource Manager to handle this scenario. See the Configuration section below for details. </p>
+
+<h3 id="components">Components</h3>
+
+<h4 id="securitymanager">SecurityManager</h4>
+
+<p>When ApplicationMaster starts, it spawns <code>SamzaAppMasterSecurityManager</code>, which runs on its separate thread. The <code>SamzaAppMasterSecurityManager</code> is responsible for periodically logging in through the given Kerberos keytab and regenerates the HDFS delegation tokens regularly. After each run, it writes new tokens on a pre-defined job specific directory on HDFS. The frequency of this process is determined by <code>yarn.token.renewal.interval.seconds</code>.</p>
+
+<p>Each container, upon start, runs a <code>SamzaContainerSecurityManager</code>. It reads from the credentials file on HDFS and refreshes its delegation tokens at the same interval.</p>
+
+<h3 id="configuration">Configuration</h3>
+
+<ol>
+<li>For the Samza job, the following job configurations are required on a YARN cluster with security enabled.
+# Job
+job.security.manager.factory=org.apache.samza.job.yarn.SamzaYarnSecurityManagerFactory</li>
+</ol>
+
+<h1 id="yarn">YARN</h1>
+
+<div class="highlight"><pre><code class="properties"><span class="na">yarn.kerberos.principal</span><span class="o">=</span><span class="s">user/localhost</span>
+<span class="na">yarn.kerberos.keytab</span><span class="o">=</span><span class="s">/etc/krb5.keytab.user</span>
+<span class="na">yarn.token.renewal.interval.seconds</span><span class="o">=</span><span class="s">86400</span></code></pre></div>
+
+<ol>
+<li>Configure the Hadoop cluster to enable Resource Manager to recreate and renew the delegation token on behalf of the application user. This will address the following 2 scenarios.</li>
+</ol>
+<div class="highlight"><pre><code class="language-text" data-lang="text">* When Application Master dies unexpectedly and needs a restart after 7 days (the default maximum lifespan a delegation token can be renewed).
+
+* When the Samza job terminates and log aggregation is turned on for the job. Node managers need to be able to upload all the local application logs to HDFS.
+
+1. Enable the resource manager as a privileged user in yarn-site.xml.
+</code></pre></div>
+<div class="highlight"><pre><code class="xml"><span class="nt"><property></span>
+ <span class="nt"><name></span>yarn.resourcemanager.proxy-user-privileges.enabled<span class="nt"></name></span>
+ <span class="nt"><value></span>true<span class="nt"></value></span>
+ <span class="nt"></property></span></code></pre></div>
+<div class="highlight"><pre><code class="language-text" data-lang="text">2. Make `yarn` as a proxy user, in core-site.xml
+</code></pre></div>
+<div class="highlight"><pre><code class="xml"><span class="nt"><property></span>
+ <span class="nt"><name></span>hadoop.proxyuser.yarn.hosts<span class="nt"></name></span>
+ <span class="nt"><value></span>*<span class="nt"></value></span>
+ <span class="nt"></property></span>
+ <span class="nt"><property></span>
+ <span class="nt"><name></span>hadoop.proxyuser.yarn.groups<span class="nt"></name></span>
+ <span class="nt"><value></span>*<span class="nt"></value></span>
+ <span class="nt"></property></span></code></pre></div>
+
+<h2 id="security-»"><a href="../operations/security.html">Security »</a></h2>
Added: samza/site/learn/tutorials/0.11/deploy-samza-job-from-hdfs.html
URL: http://svn.apache.org/viewvc/samza/site/learn/tutorials/0.11/deploy-samza-job-from-hdfs.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/tutorials/0.11/deploy-samza-job-from-hdfs.html (added)
+++ samza/site/learn/tutorials/0.11/deploy-samza-job-from-hdfs.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,210 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Deploying a Samza job from HDFS</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/tutorials/latest/deploy-samza-job-from-hdfs.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Deploying a Samza job from HDFS</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>This tutorial uses <a href="../../../startup/hello-samza/0.11/">hello-samza</a> to illustrate how to run a Samza job if you want to publish the Samza job’s .tar.gz package to HDFS.</p>
+
+<h3 id="upload-the-package">Upload the package</h3>
+
+<div class="highlight"><pre><code class="bash">hadoop fs -put ./target/hello-samza-0.11.0-dist.tar.gz /path/for/tgz</code></pre></div>
+
+<h3 id="add-hdfs-configuration">Add HDFS configuration</h3>
+
+<p>Put the hdfs-site.xml file of your cluster into ~/.samza/conf directory (The same place as the yarn-site.xml). If you set HADOOP_CONF_DIR, put the hdfs-site.xml in your configuration directory if the hdfs-site.xml is not there.</p>
+
+<h3 id="change-properties-file">Change properties file</h3>
+
+<p>Change the yarn.package.path in the properties file to your HDFS location.</p>
+
+<div class="highlight"><pre><code class="jproperties"><span class="na">yarn.package.path</span><span class="o">=</span><span class="s">hdfs://<hdfs name node ip>:<hdfs name node port>/path/to/tgz</span></code></pre></div>
+
+<p>Then you should be able to run the Samza job as described in <a href="../../../startup/hello-samza/0.11/">hello-samza</a>.</p>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/tutorials/latest/deploy-samza-job-from-hdfs.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/tutorials/0.11/deploy-samza-to-CDH.html
URL: http://svn.apache.org/viewvc/samza/site/learn/tutorials/0.11/deploy-samza-to-CDH.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/tutorials/0.11/deploy-samza-to-CDH.html (added)
+++ samza/site/learn/tutorials/0.11/deploy-samza-to-CDH.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,228 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Deploy Samza Job To CDH</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/tutorials/latest/deploy-samza-to-CDH.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Deploy Samza Job To CDH</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>The tutorial assumes you have successfully run <a href="../../../startup/hello-samza/0.11/">hello-samza</a> and now you want to deploy the job to your Cloudera Data Hub (<a href="http://www.cloudera.com/content/cloudera/en/products-and-services/cdh.html">CDH</a>). This tutorial is based on CDH 5.4.0 and uses hello-samza as the example job.</p>
+
+<h3 id="compile-package-for-cdh-5.4.0">Compile Package for CDH 5.4.0</h3>
+
+<p>We need to use a specific compile option to build hello-samza package for CDH 5.4.0</p>
+
+<div class="highlight"><pre><code class="bash">mvn clean package -Denv<span class="o">=</span>cdh5.4.0</code></pre></div>
+
+<h3 id="upload-package-to-cluster">Upload Package to Cluster</h3>
+
+<p>There are a few ways of uploading the package to the cluster’s HDFS. If you do not have the job package in your cluster, <strong>scp</strong> from you local machine to the cluster. Then run</p>
+
+<div class="highlight"><pre><code class="bash">hadoop fs -put path/to/hello-samza-0.11.0-dist.tar.gz /path/for/tgz</code></pre></div>
+
+<h3 id="get-deloying-scripts">Get Deloying Scripts</h3>
+
+<p>Untar the job package (assume you will run from the current directory)</p>
+
+<div class="highlight"><pre><code class="bash">tar -xvf path/to/samza-job-package-0.11.0-dist.tar.gz -C ./</code></pre></div>
+
+<h3 id="add-package-path-to-properties-file">Add Package Path to Properties File</h3>
+
+<div class="highlight"><pre><code class="bash">vim config/wikipedia-parser.properties</code></pre></div>
+
+<p>Change the yarn package path:</p>
+
+<div class="highlight"><pre><code class="jproperties"><span class="na">yarn.package.path</span><span class="o">=</span><span class="s">hdfs://<hdfs name node ip>:<hdfs name node port>/path/to/tgz</span></code></pre></div>
+
+<h3 id="set-yarn-environment-variable">Set Yarn Environment Variable</h3>
+
+<div class="highlight"><pre><code class="bash"><span class="nb">export </span><span class="nv">HADOOP_CONF_DIR</span><span class="o">=</span>/etc/hadoop/conf</code></pre></div>
+
+<h3 id="run-samza-job">Run Samza Job</h3>
+
+<div class="highlight"><pre><code class="bash">bin/run-job.sh --config-factory<span class="o">=</span>org.apache.samza.config.factories.PropertiesConfigFactory --config-path<span class="o">=</span>file://<span class="nv">$PWD</span>/config/wikipedia-parser.properties</code></pre></div>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/tutorials/latest/deploy-samza-to-CDH.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/tutorials/0.11/index.html
URL: http://svn.apache.org/viewvc/samza/site/learn/tutorials/0.11/index.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/tutorials/0.11/index.html (added)
+++ samza/site/learn/tutorials/0.11/index.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,217 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Tutorials</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/tutorials/latest/index.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Tutorials</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p><a href="remote-debugging-samza.html">Remote Debugging with Samza</a></p>
+
+<p><a href="deploy-samza-job-from-hdfs.html">Deploying a Samza Job from HDFS</a></p>
+
+<p><a href="deploy-samza-to-CDH.html">Deoloy Samza to CDH</a></p>
+
+<p><a href="run-in-multi-node-yarn.html">Run Hello-samza in Multi-node YARN</a></p>
+
+<p><a href="run-hello-samza-without-internet.html">Run Hello-samza without Internet</a></p>
+
+<p><a href="samza-rest-getting-started.html">Getting Started with Samza REST</a></p>
+
+<p><a href="samza-async-user-guide.html">Samza Async API and Multithreading User Guide</a></p>
+
+<!-- TODO a bunch of tutorials
+[Log Walkthrough](log-walkthrough.html)
+<a href="configuring-kafka-system.html">Configuring a Kafka System</a><br/>
+<a href="joining-streams.html">Joining Streams</a><br/>
+<a href="sort-stream.html">Sorting a Stream</a><br/>
+<a href="group-by-count.html">Group-by and Counting</a><br/>
+<a href="initialize-close.html">Initializing and Closing</a><br/>
+<a href="windowing.html">Windowing</a><br/>
+<a href="committing.html">Committing</a><br/>
+–>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/tutorials/latest/index.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>