You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by xi...@apache.org on 2016/10/19 17:21:45 UTC
svn commit: r1765686 [24/42] - in /samza/site: ./ archive/ community/
contribute/ img/0.11/ img/0.11/learn/ img/0.11/learn/documentation/
img/0.11/learn/documentation/comparisons/
img/0.11/learn/documentation/container/ img/0.11/learn/documentation/int...
Added: samza/site/learn/documentation/0.11/container/coordinator-stream.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/container/coordinator-stream.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/container/coordinator-stream.html (added)
+++ samza/site/learn/documentation/0.11/container/coordinator-stream.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,324 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Coordinator Stream</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/documentation/latest/container/coordinator-stream.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Coordinator Stream</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>Samza job is completely driven by the job configuration. Thus, job configurations tend to be pretty large. In order to easily serialize such large configs and persist them between job executions, Samza writes all configurations to a durable stream called the <em>Coordinator Stream</em> when a job is submitted.</p>
+
+<p>A Coordinator Stream is single partitioned stream to which the configurations are written to. It shares the same characteristics as any input stream that can be configured in Samza - ordered, replayable and fault-tolerant. The stream will contain three major types of messages:</p>
+
+<ol>
+<li>Job configuration messages</li>
+<li>Task changelog partition assignment messages</li>
+<li>Container locality message</li>
+</ol>
+
+<h3 id="coordinator-stream-naming">Coordinator Stream Naming</h3>
+
+<p>The naming convention is very similar to that of the checkpoint topic that get’s created.</p>
+<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="s">"__samza_coordinator_%s_%s"</span> <span class="n">format</span> <span class="o">(</span><span class="n">jobName</span><span class="o">.</span><span class="na">replaceAll</span><span class="o">(</span><span class="s">"_"</span><span class="o">,</span> <span class="s">"-"</span><span class="o">),</span> <span class="n">jobId</span><span class="o">.</span><span class="na">replaceAll</span><span class="o">(</span><span class="s">"_"</span><span class="o">,</span> <span class="s">"-"</span><span class="o">))</span>
+</code></pre></div>
+<h3 id="coordinator-stream-message-model">Coordinator Stream Message Model</h3>
+
+<p>Coordinator stream messages are modeled as key/value pairs. The key is a list of well defined fields: <em>version</em>, <em>type</em>, and <em>key</em>. The value is a <em>map</em>. There are some pre-defined fields (such as timestamp, host, etc) for the value map, which are common to all messages.</p>
+
+<p>The full structure for a CoordinatorStreamMessage is:</p>
+<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="err">key</span> <span class="err">=></span> <span class="p">[</span><span class="s2">"<version-number>"</span><span class="p">,</span> <span class="s2">"<message-type>"</span><span class="p">,</span> <span class="s2">"<key>"</span><span class="p">]</span>
+
+<span class="err">message</span> <span class="err">=></span> <span class="p">{</span>
+ <span class="nt">"host"</span><span class="p">:</span> <span class="s2">"<hostname>"</span><span class="p">,</span>
+ <span class="nt">"username"</span><span class="p">:</span> <span class="s2">"<username>"</span><span class="p">,</span>
+ <span class="nt">"source"</span><span class="p">:</span> <span class="s2">"<source-for-this-message>"</span><span class="p">,</span>
+ <span class="nt">"timestamp"</span><span class="p">:</span> <span class="err"><timestamp-of-the-message></span><span class="p">,</span>
+ <span class="nt">"values"</span><span class="p">:</span> <span class="p">{</span> <span class="p">}</span>
+<span class="p">}</span>
+</code></pre></div>
+<p>The messages are essentially serialized and transmitted over the wire as JSON blobs. Hence, for serialization to work correctly, it is very important to not have any unnecessary white spaces. The white spaces in the above JSON blob have been shown for legibility only.</p>
+
+<p>The most important fields are type, key, and values:</p>
+
+<ul>
+<li>type - defines the kind of message</li>
+<li>key - defines a key to associate with the values</li>
+<li>values map - defined on a per-message-type basis, and defines a set of values associated with the type</li>
+</ul>
+
+<p>The coordinator stream messages that are currently supported are listed below:
+<style>
+ table th, table td {
+ text-align: left;
+ vertical-align: top;
+ padding: 12px;
+ border-bottom: 1px solid #ccc;
+ border-top: 1px solid #ccc;
+ border-left: 0;
+ border-right: 0;
+ }</p>
+<div class="highlight"><pre><code class="language-text" data-lang="text"> table td.property, table td.default {
+ white-space: nowrap;
+ }
+
+ table th {
+ background-color: #eee;
+ }
+</code></pre></div>
+<p></style>
+<table>
+ <tr>
+ <th>Message</th>
+ <th>Type</th>
+ <th>Key</th>
+ <th>Values Map</th>
+ </tr>
+ <tr>
+ <td> Configuration Message <br />
+ (Applies to all configuration <br />
+ options listed in <a href="../jobs/configuration-table.html">Configuration</a>) </td>
+ <td> set-config </td>
+ <td> <config-name> </td>
+ <td> ‘value’ => <config-value> </td>
+ </tr>
+ <tr>
+ <td> Task-ChangelogPartition Assignment Message </td>
+ <td> set-changelog </td>
+ <td> <<a href="../api/org/apache/samza/container/TaskName.java">TaskName</a>> </td>
+ <td> ‘partition’ => <Changelog-Partition-Id>
+ </td>
+ </tr>
+ <tr>
+ <td> Container Locality Message </td>
+ <td> set-container-host-assignment </td>
+ <td> <Container-Id> </td>
+ <td> ‘hostname’ => <HostName>
+ </td>
+ </tr>
+</table></p>
+
+<h3 id="coordinator-stream-writer">Coordinator Stream Writer</h3>
+
+<p>Samza provides a command line tool to write Job Configuration messages to the coordinator stream. The tool can be used as follows:</p>
+
+<div class="highlight"><pre><code class="bash">samza-example/target/bin/run-coordinator-stream-writer.sh <span class="se">\</span>
+ --config-path<span class="o">=</span>file:///path/to/job/config.properties <span class="se">\</span>
+ --type <span class="nb">set</span>-config <span class="se">\</span>
+ --key job.container.count <span class="se">\</span>
+ --value 8</code></pre></div>
+
+<h2 id="job-coordinator"><a name="JobCoordinator"></a>Job Coordinator</h2>
+
+<p>The Job Coordinator bootstraps configuration from the coordinator stream each time upon job start-up. It periodically catches up with any new data written to the coordinator stream and updates the <em>Job Model</em>.</p>
+
+<p>Job Model is the data model used to represent a Samza job, which also incorporates the Job configuration. The hierarchy of a Samza job - job has containers, and each of the containers has tasks - is encapsulated in the Job Model, along with relevant information such as container id, task names, partition information, etc.</p>
+
+<p>The Job Coordinator exposes the Job Model and Job Configuration via an HTTP service. The URL for the Job Coordinator’s HTTP service is passed as an environment variable to the Samza Containers when the containers are launched. Containers may write meta-information, such as locality - the hostname of the machine on which the container is running. However, they will read the Job Model and Configuration by querying the Job Coordinator via the HTTP service.</p>
+
+<p>Thus, Job Coorindator is the single component that has the latest view of the entire job status. This is very useful as it allows us to extend functionality of the Job Coordinator, in the future, to manage the lifecycle of the job (such as start/stop container, modify task assignment etc).</p>
+
+<h3 id="job-coordinator-availability">Job Coordinator Availability</h3>
+
+<p>The Job Coordinator resides in the same container as the Samza Application Master. Thus, the availability of the Job Coordinator is tied to the availability of the Application Master (AM) in the Yarn cluster. The Samza containers are started only after initializing the Job Coordinator from the Coordinator Stream. In stable condition, when the Samza container comes up, it should be able to read the JobModel from the Job Coordinator without timing out.</p>
+
+<h2 id="benefits-of-coordinator-stream-model">Benefits of Coordinator Stream Model</h2>
+
+<p>Writing the configuration to a durable stream opens the door for Samza to do a couple of things:</p>
+
+<ol>
+<li>Removes the size-bound on the Job configuration</li>
+<li>Exposes job-related configuration and metadata to the containers using a standard data model and communication interface (See <a href="#JobCoordinator">Job Coordinator</a> for details)</li>
+<li>Certain configurations should only be set one time. Changing them in future deployment amounts to resetting the entire state of the job because it may re-shuffle input partitions to the containers. For example, changing <a href="../api/javadocs/org/apache/samza/container/grouper/stream/SystemStreamPartitionGrouper.java">SystemStreamPartitionGrouper</a> on a stateful Samza job would inter-mingle state from different StreamTasks in a single changelog partition. Without persistent configuration, there is no easy way to check whether a job’s current configuration is valid or not.</li>
+<li>Job configuration can be dynamically changed by writing to the Coorinator Stream. This can enable features that require the job to be reactive to configuration change (eg. host-affinity, auto-scaling, dynamic reconfiguration etc).</li>
+<li>Provides a unified view of the job state, enabling Samza with more powerful ways of controlling container controls (See <a href="#JobCoordinator">Job Coordinator</a> for details)</li>
+<li>Enables future design of Job Coordinator fail-over since it serves as a single source of truth of the current job state</li>
+</ol>
+
+<p>For other interesting features that can leverage this model, please refer the <a href="https://issues.apache.org/jira/secure/attachment/12670650/DESIGN-SAMZA-348-1.pdf">design document</a>.</p>
+
+<h2 id="event-loop-»"><a href="event-loop.html">Event Loop »</a></h2>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/documentation/latest/container/coordinator-stream.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/documentation/0.11/container/event-loop.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/container/event-loop.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/container/event-loop.html (added)
+++ samza/site/learn/documentation/0.11/container/event-loop.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,246 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - Event Loop</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/documentation/latest/container/event-loop.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>Event Loop</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>The event loop orchestrates <a href="streams.html">reading and processing messages</a>, <a href="checkpointing.html">checkpointing</a>, <a href="windowing.html">windowing</a> and <a href="metrics.html">flushing metrics</a> among tasks. </p>
+
+<p>By default Samza uses a single thread in each <a href="samza-container.html">container</a> to run the tasks. This fits CPU-bound jobs well; to get more CPU processors, simply add more containers. The single thread execution also simplifies sharing task state and resource management.</p>
+
+<p>For IO-bound jobs, Samza supports finer-grained parallelism for both synchronous and asynchronous tasks. For synchronous tasks (<a href="../api/javadocs/org/apache/samza/task/StreamTask.html">StreamTask</a> and <a href="../api/javadocs/org/apache/samza/task/WindowableTask.html">WindowableTask</a>), you can schedule them to run in parallel by configuring the build-in thread pool <a href="../jobs/configuration-table.html">job.container.thread.pool.size</a>. This fits the blocking-IO task scenario. For asynchronous tasks (<a href="../api/javadocs/org/apache/samza/task/AsyncStreamTask.html">AsyncStreamTask</a>), you can make async IO calls and trigger callbacks upon completion. The finest degree of parallelism Samza provides is within a task, and is configured by <a href="../jobs/configuration-table.html">task.max.concurrency</a>.</p>
+
+<p>The latest version of Samza is thread-safe. You can safely access your job’s state in <a href="state-management.html">key-value store</a>, write messages and checkpoint offset in the task threads. If you have other data shared among tasks, such as global variables or static data, it is not thread safe if the data can be accessed concurrently by multiple threads, e.g. StreamTask running in the configured thread pool with more than one threads. For states within a task, such as member variables, Samza guarantees the mutual exclusiveness of process, window and commit so there will be no concurrent modifications among these operations and any state change from one operation will be fully visible to the others. </p>
+
+<h3 id="event-loop-internals">Event Loop Internals</h3>
+
+<p>A container may have multiple <a href="../api/javadocs/org/apache/samza/system/SystemConsumer.html">SystemConsumers</a> for consuming messages from different input systems. Each SystemConsumer reads messages on its own thread, but writes messages into a shared in-process message queue. The container uses this queue to funnel all of the messages into the event loop.</p>
+
+<p>The event loop works as follows:</p>
+
+<ol>
+<li>Choose a message from the incoming message queue;</li>
+<li>Schedule the appropriate <a href="samza-container.html">task instance</a> to process the message;</li>
+<li>Schedule window() on the task instance to run if it implements WindowableTask, and the window timer has been triggered;</li>
+<li>Send any output from the process() and window() calls to the appropriate <a href="../api/javadocs/org/apache/samza/system/SystemProducer.html">SystemProducers</a>;</li>
+<li>Write checkpoints and flush the state stores for any tasks whose <a href="checkpointing.html">commit interval</a> has elapsed.</li>
+<li>Block if all task instances are busy with processing outstanding messages, windowing or checkpointing.</li>
+</ol>
+
+<p>The container does this, in a loop, until it is shut down.</p>
+
+<h3 id="semantics-for-synchronous-tasks-v.s.-asynchronous-tasks">Semantics for Synchronous Tasks v.s. Asynchronous Tasks</h3>
+
+<p>The semantics of the event loop differs when running synchronous tasks and asynchronous tasks:</p>
+
+<ul>
+<li>For synchronous tasks (StreamTask and WindowableTask), process() and window() will run on the single main thread by default. You can configure job.container.thread.pool.size to be greater than 1, and event loop will schedule the process() and window() to run in the thread pool.<br></li>
+<li>For Asynchronous tasks (AsyncStreamTask), processAsync() will always be invoked in a single thread, while callbacks can be triggered from a different user thread. </li>
+</ul>
+
+<p>In both cases, the default concurrency within a task is 1, meaning at most one outstanding message in processing per task. This guarantees in-order message processing in a topic partition. You can further increase it by configuring task.max.concurrency to be greater than 1. This allows multiple outstanding messages to be processed in parallel by a task. This option increases the parallelism within a task, but may result in out-of-order processing and completion.</p>
+
+<p>The following semantics are guaranteed in any of the above cases (for happens-before semantics, see <a href="https://docs.oracle.com/javase/tutorial/essential/concurrency/memconsist.html">here</a>):</p>
+
+<ul>
+<li>If task.max.concurrency = 1, each message process completion in a task is guaranteed to happen-before the next invocation of process()/processAsync() of the same task. If task.max.concurrency > 1, there is no such happens-before constraint and user should synchronize access to any shared/global variables in the Task..</li>
+<li>WindowableTask.window() is called when no invocations to process()/processAsync() are pending and no new process()/processAsync() invocations can be scheduled until it completes. Therefore, a guarantee that all previous process()/processAsync() invocations happen before an invocation of WindowableTask.window(). An invocation to WindowableTask.window() is guaranteed to happen-before any subsequent process()/processAsync() invocations. The Samza engine is responsible for ensuring that window is invoked in a timely manner.</li>
+<li>Checkpointing is guaranteed to only cover events that are fully processed. It happens only when there are no pending process()/processAsync() or WindowableTask.window() invocations. All preceding invocations happen-before checkpointing and checkpointing happens-before all subsequent invocations.</li>
+</ul>
+
+<p>More details and examples can be found in <a href="../../../tutorials/0.11/samza-async-user-guide.html">Samza Async API and Multithreading User Guide</a>.</p>
+
+<h3 id="lifecycle">Lifecycle</h3>
+
+<p>The only way in which a developer can hook into a SamzaContainer’s lifecycle is through the standard InitableTask, ClosableTask, StreamTask/AsyncStreamTask, and WindowableTask. In cases where pluggable logic needs to be added to wrap a StreamTask, the StreamTask can be wrapped by another StreamTask implementation that handles the custom logic before calling into the wrapped StreamTask.</p>
+
+<p>A concrete example is a set of StreamTasks that all want to share the same try/catch logic in their process() method. A StreamTask can be implemented that wraps the original StreamTasks, and surrounds the original process() call with the appropriate try/catch logic. For more details, see <a href="https://issues.apache.org/jira/browse/SAMZA-437">this discussion</a>.</p>
+
+<h2 id="metrics-»"><a href="metrics.html">Metrics »</a></h2>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/documentation/latest/container/event-loop.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/documentation/0.11/container/jmx.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/container/jmx.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/container/jmx.html (added)
+++ samza/site/learn/documentation/0.11/container/jmx.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,209 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <title>Samza - JMX</title>
+ <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/>
+ <link href="/css/bootstrap.min.css" rel="stylesheet"/>
+ <link href="/css/font-awesome.min.css" rel="stylesheet"/>
+ <link href="/css/main.css" rel="stylesheet"/>
+ <link href="/css/syntax.css" rel="stylesheet"/>
+ <link rel="icon" type="image/png" href="/img/samza-icon.png">
+ <script src="/js/jquery-1.11.1.min.js"></script>
+ </head>
+ <body>
+ <div class="wrapper">
+ <div class="wrapper-content">
+
+ <div class="masthead">
+ <div class="container">
+ <div class="masthead-logo">
+ <a href="/" class="logo">samza</a>
+ </div>
+ <div class="masthead-icons">
+ <div class="pull-right">
+ <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a>
+ <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a>
+ <a href="https://twitter.com/samzastream" target="_blank"><i class="fa fa-twitter masthead-icon"></i></a>
+ <!-- this icon only shows in versioned pages -->
+
+
+
+
+ <a href="http://samza.apache.org/learn/documentation/latest/container/jmx.html"><i id="switch-version-button"></i></a>
+ <!-- links for the navigation bar -->
+
+
+ </div>
+ </div>
+ </div><!-- /.container -->
+ </div>
+
+ <div class="container">
+ <div class="menu">
+ <h1><i class="fa fa-rocket"></i> Getting Started</h1>
+ <ul>
+ <li><a href="/startup/hello-samza/0.11">Hello Samza</a></li>
+ <li><a href="/startup/download">Download</a></li>
+ </ul>
+
+ <h1><i class="fa fa-book"></i> Learn</h1>
+ <ul>
+ <li><a href="/learn/documentation/0.11">Documentation</a></li>
+ <li><a href="/learn/documentation/0.11/jobs/configuration-table.html">Configuration</a></li>
+ <li><a href="/learn/documentation/0.11/container/metrics-table.html">Metrics</a></li>
+ <li><a href="/learn/documentation/0.11/api/javadocs/">Javadocs</a></li>
+ <li><a href="/learn/tutorials/0.11">Tutorials</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ">FAQ</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza">Wiki</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876">Papers & Talks</a></li>
+ <li><a href="http://blogs.apache.org/samza">Blog</a></li>
+ </ul>
+
+ <h1><i class="fa fa-comments"></i> Community</h1>
+ <ul>
+ <li><a href="/community/mailing-lists.html">Mailing Lists</a></li>
+ <li><a href="/community/irc.html">IRC</a></li>
+ <li><a href="https://issues.apache.org/jira/browse/SAMZA">Bugs</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">Powered by</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem">Ecosystem</a></li>
+ <li><a href="/community/committers.html">Committers</a></li>
+ </ul>
+
+ <h1><i class="fa fa-code"></i> Contribute</h1>
+ <ul>
+ <li><a href="/contribute/rules.html">Rules</a></li>
+ <li><a href="/contribute/coding-guide.html">Coding Guide</a></li>
+ <li><a href="/contribute/projects.html">Projects</a></li>
+ <li><a href="/contribute/design-documents.html">Design Documents</a></li>
+ <li><a href="/contribute/code.html">Code</a></li>
+ <li><a href="https://reviews.apache.org/groups/samza">Review Board</a></li>
+ <li><a href="/contribute/tests.html">Tests</a></li>
+ </ul>
+
+ <h1><i class="fa fa-history"></i> Archive</h1>
+ <ul>
+ <li><a href="/archive/index.html#latest">latest</a></li>
+ <li><a href="/archive/index.html#10">0.10</a></li>
+ <li><a href="/archive/index.html#09">0.9</a></li>
+ <li><a href="/archive/index.html#08">0.8</a></li>
+ <li><a href="/archive/index.html#07">0.7</a></li>
+ </ul>
+ </div>
+
+ <div class="content">
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<h2>JMX</h2>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<p>Samza’s containers and YARN ApplicationMaster enable <a href="http://docs.oracle.com/javase/tutorial/jmx/">JMX</a> by default. JMX can be used for managing the JVM; for example, you can connect to it using <a href="http://docs.oracle.com/javase/7/docs/technotes/guides/management/jconsole.html">jconsole</a>, which is included in the JDK.</p>
+
+<p>You can tell Samza to publish its internal <a href="metrics.html">metrics</a>, and any custom metrics you define, as JMX MBeans. To enable this, set the following properties in your job configuration:</p>
+
+<div class="highlight"><pre><code class="jproperties"><span class="c"># Define a Samza metrics reporter called "jmx", which publishes to JMX</span>
+<span class="na">metrics.reporter.jmx.class</span><span class="o">=</span><span class="s">org.apache.samza.metrics.reporter.JmxReporterFactory</span>
+
+<span class="c"># Use it (if you have multiple reporters defined, separate them with commas)</span>
+<span class="na">metrics.reporters</span><span class="o">=</span><span class="s">jmx</span></code></pre></div>
+
+<p>JMX needs to be configured to use a specific port, but in a distributed environment, there is no way of knowing in advance which ports are available on the machines running your containers. Therefore Samza chooses the JMX port randomly. If you need to connect to it, you can find the port by looking in the container’s logs, which report the JMX server details as follows:</p>
+<div class="highlight"><pre><code class="language-text" data-lang="text">2014-06-02 21:50:17 JmxServer [INFO] According to InetAddress.getLocalHost.getHostName we are samza-grid-1234.example.com
+2014-06-02 21:50:17 JmxServer [INFO] Started JmxServer registry port=50214 server port=50215 url=service:jmx:rmi://localhost:50215/jndi/rmi://localhost:50214/jmxrmi
+2014-06-02 21:50:17 JmxServer [INFO] If you are tunneling, you might want to try JmxServer registry port=50214 server port=50215 url=service:jmx:rmi://samza-grid-1234.example.com:50215/jndi/rmi://samza-grid-1234.example.com:50214/jmxrmi
+</code></pre></div>
+<h2 id="jobrunner-»"><a href="../jobs/job-runner.html">JobRunner »</a></h2>
+
+
+ </div>
+ </div>
+
+ </div><!-- /.wrapper-content -->
+ </div><!-- /.wrapper -->
+
+ <div class="footer">
+ <div class="container">
+ <!-- nothing for now. -->
+ </div>
+ </div>
+
+
+ <script>
+ $( document ).ready(function() {
+ if ( $.fn.urlExists( "/learn/documentation/latest/container/jmx.html" ) ) {
+ $("#switch-version-button").addClass("fa fa-history masthead-icon");
+ }
+ });
+
+ /* a function to test whether the url exists or not */
+ (function( $ ) {
+ $.fn.urlExists = function(url) {
+ var http = new XMLHttpRequest();
+ http.open('HEAD', url, false);
+ http.send();
+ return http.status != 404;
+ };
+ }( jQuery ));
+ </script>
+
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-43122768-1', 'apache.org');
+ ga('send', 'pageview');
+
+ </script>
+ </body>
+</html>
Added: samza/site/learn/documentation/0.11/container/metrics-table.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.11/container/metrics-table.html?rev=1765686&view=auto
==============================================================================
--- samza/site/learn/documentation/0.11/container/metrics-table.html (added)
+++ samza/site/learn/documentation/0.11/container/metrics-table.html Wed Oct 19 17:21:41 2016
@@ -0,0 +1,889 @@
+<!DOCTYPE html>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta charset="utf-8">
+ <title>Samza Metrics Reference</title>
+ <style type="text/css">
+ body {
+ font-family: "Helvetica Neue",Helvetica,Arial,sans-serif;
+ font-size: 14px;
+ line-height: 22px;
+ color: #333;
+ background-color: #fff;
+ }
+
+ table {
+ border-collapse: collapse;
+ margin: 1em 0;
+ }
+
+ table th, table td {
+ text-align: left;
+ vertical-align: top;
+ padding: 12px;
+ border-bottom: 1px solid #ccc;
+ border-top: 1px solid #ccc;
+ border-left: 0;
+ border-right: 0;
+ }
+
+ table td.property, table td.default {
+ white-space: nowrap;
+ }
+
+ table th.section {
+ background-color: #eee;
+ }
+
+ table th.section .subtitle {
+ font-weight: normal;
+ }
+ code, a.property {
+ font-family: monospace;
+ }
+ code {
+ font-family: 'DejaVu Sans Mono', monospace;
+ font-size: 14px;
+ padding-top: 4px;
+ margin-top: 8px;
+ line-height: 1.4em;
+ background-color: rgba(220,220,220,0.4);
+ font-weight: lighter;
+ }
+ span.system, span.stream, span.topic, span.partition, span.source, span.host, span.port, span.store, span.task {
+ padding: 1px;
+ margin: 1px;
+ border-width: 1px;
+ border-style: solid;
+ border-radius: 4px;
+ }
+
+ span.system {
+ background-color: #ddf;
+ border-color: #bbd;
+ }
+
+ span.stream, span.topic {
+ background-color: #dfd;
+ border-color: #bdb;
+ }
+
+ span.partition {
+ background-color: #dff;
+ border-color: #bdd;
+ }
+
+ span.source {
+ background-color: #fdd;
+ border-color: #dbb;
+ }
+
+ span.host {
+ background-color: #bfd;
+ border-color: #adc;
+ }
+
+ span.port {
+ background-color: #bfb;
+ border-color: #ada;
+ }
+
+ span.store {
+ background-color: #fdf;
+ border-color: #dbd;
+ }
+
+ span.task {
+ background-color: #fbd;
+ border-color: #d9b;
+ }
+ </style>
+</head>
+
+<body>
+<h1>Samza Metrics Reference</h1>
+<p>The following table lists all the metrics emited by Samza.</p>
+<p>Metrics are grouped by group names usually representing classes that are adding their own metrics to the registry. Those are the following:</p>
+<ul>
+ <li><a href="#samza-container-metrics">SamzaContainerMetrics</a></li>
+ <li><a href="#job-coordinator">job-coordinator</a></li>
+ <li><a href="#task-instance-metrics">TaskInstanceMetrics</a></li>
+ <li><a href="#offset-manager-metrics">OffsetManagerMetrics</a></li>
+ <li><a href="#jvm-metrics">JvmMetrics</a></li>
+ <li><a href="#system-consumers-metrics">SystemConsumersMetrics</a></li>
+ <li><a href="#system-producers-metrics">SystemProducersMetrics</a></li>
+ <li><a href="#container-process-manager-metrics">ContainerProcessManagerMetrics</a></li>
+ <li><a href="#samza-app-master-metrics">SamzaAppMasterMetrics</a></li>
+ <li><a href="#kafka-system-consumer-metrics">KafkaSystemConsumerMetrics</a></li>
+ <li><a href="#kafka-system-producer-metrics">KafkaSystemProducerMetrics</a></li>
+ <li><a href="#key-value-storage-engine-metrics">KeyValueStorageEngineMetrics</a></li>
+ <li><a href="#key-value-store-metrics">KeyValueStoreMetrics</a></li>
+ <li><a href="#logged-store-metrics">LoggedStoreMetrics</a></li>
+ <li><a href="#serialized-key-value-store-metrics">SerializedKeyValueStoreMetrics</a></li>
+ <li><a href="#cached-store-metrics">CachedStoreMetrics</a></li>
+ <li><a href="#round-robin-chooser-metrics">RoundRobinChooserMetrics</a></li>
+ <li><a href="#batching-chooser-metrics">BatchingChooserMetrics</a></li>
+ <li><a href="#bootstrapping-chooser-metrics">BootstrappingChooserMetrics</a></li>
+ <li><a href="#hdfs-system-producer-metrics">HdfsSystemProducerMetrics</a></li>
+ <li><a href="#elasticsearch-system-producer-metrics">ElasticsearchSystemProducerMetrics</a></li>
+</ul>
+<p>Words highlighted like <span class="system">this</span> are placeholders for your own variable names defined in configuration file or system variables defined while starting the job.</p>
+<p id="average-time" style="color: #00a">Note: Average time is calculated for the current time window (set to 300 seconds)</p>
+<table>
+ <tbody>
+ <tr><th>Name</th><th>Description</th></tr>
+ <tr>
+ <th colspan="2" class="section" id="samza-container-metrics">org.apache.samza.container.SamzaContainerMetrics</th>
+ </tr>
+ <tr>
+ <td>process-calls</td>
+ <td>Number of times process method is called</td>
+ </tr>
+ <tr>
+ <td>process-ns</td>
+ <td><a href="#average-time">Average time</a> spent for processing messages</td>
+ </tr>
+ <tr>
+ <td>process-envelopes</td>
+ <td>Number of message envelopes processed</td>
+ </tr>
+ <tr>
+ <td>process-null-envelopes</td>
+ <td>Number of times no message envelope was available</td>
+ </tr>
+ <tr>
+ <td>choose-ns</td>
+ <td><a href="#average-time">Average time</a> spent for choosing the envelope to process</td>
+ </tr>
+ <tr>
+ <td>commit-calls</td>
+ <td>Number of commit method calls</td>
+ </tr>
+ <tr>
+ <td>commit-ns</td>
+ <td><a href="#average-time">Average time</a> spent for commiting tasks states.</td>
+ </tr>
+ <tr>
+ <td>event-loop-utilization</td>
+ <td>Number representing duty-cycle (active time in event loop divided by the total time spent in the event loop)</td>
+ </tr>
+ <tr>
+ <td>window-calls</td>
+ <td>Number of window method calls</td>
+ </tr>
+ <tr>
+ <td>window-ns</td>
+ <td><a href="#average-time">Average time</a> spent for windowing tasks</td>
+ </tr>
+ <tr>
+ <td>block-ns</td>
+ <td><a href="#average-time">Average time</a> run loop was blocked (for multithreaded processing)</td>
+ </tr>
+ <tr>
+ <td><span class="task">task-name</span>-<span class="store">store-name</span>-restore-time</td>
+ <td>Time taken to restore task stores (per task store)</td>
+ </tr>
+ <tr>
+ <td>disk-usage-bytes</td>
+ <td>Total disk space size used by key-value stores (in bytes)</td>
+ </tr>
+ <tr>
+ <td>disk-quota-bytes</td>
+ <td>Disk memory usage quota for key-value stores (in bytes)</td>
+ </tr>
+ <tr>
+ <td>executor-work-factor</td>
+ <td>Current work factor in use</td>
+ </tr>
+ <tr>
+ <td>physical-memory-mb</td>
+ <td>The physical memory used by the Samza container process (native + on heap) (in megabytes)</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="job-coordinator">job-coordinator</th>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="stream">stream</span>-partitionCount</td>
+ <td>The difference in the partition count from the initial state for a particular system and stream (enabled if job.coordinator.monitor-partition-change configured to true)</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="task-instance-metrics">org.apache.samza.container.TaskInstanceMetrics<br/><span style="font-weight: normal;margin-left:40px; color : #f00">- Following metrics apply to each TaskInstance</span></th>
+ </tr>
+ <tr>
+ <td>commit-calls</td>
+ <td>Number of commit method calls</td>
+ </tr>
+ <tr>
+ <td>flush-calls</td>
+ <td>Number of times underlying SystemProducers were flushed</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="stream">stream</span>-<span class="partition">partition</span>-offset</td>
+ <td>Latest offsets for each SystemStreamPartition of the task instance</td>
+ </tr>
+ <tr>
+ <td>messages-actually-processed</td>
+ <td>Number of messages actually processed by a task</td>
+ </tr>
+ <tr>
+ <td>process-calls</td>
+ <td>Number of proccess method calls</td>
+ </tr>
+ <tr>
+ <td>send-calls</td>
+ <td>Number of send method calls (representing number of messages that were sent to the underlying SystemProducers)</td>
+ </tr>
+ <tr>
+ <td>window-calls</td>
+ <td>Number of times windowing the task</td>
+ </tr>
+ <tr>
+ <td>pending-messages</td>
+ <td>Current number of pending messages in a queue (for multithreaded processing)</td>
+ </tr>
+ <tr>
+ <td>messages-in-flight</td>
+ <td>Number of messages in processing (for multithreaded processing)</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="offset-manager-metrics">org.apache.samza.checkpoint.OffsetManagerMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="stream">stream</span>-<span class="partition">partition</span>-checkpointed-offset</td>
+ <td>Latest checkpointed offsets for each SystemStreamPartition of the task instance</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="jvm-metrics">org.apache.samza.metrics.JvmMetrics</th>
+ </tr>
+ <tr>
+ <td>gc-count</td>
+ <td>Number of GC calls</td>
+ </tr>
+ <tr>
+ <td>gc-time-millis</td>
+ <td>Total time spent in GC (in milliseconds)</td>
+ </tr>
+ <tr>
+ <td>mem-heap-comitted-mb</td>
+ <td>Size of comitted heap memory (in megabytes)</td>
+ </tr>
+ <tr>
+ <td>mem-heap-max-mb</td>
+ <td>Size of maximum heap memory (in megabytes)</td>
+ </tr>
+ <tr>
+ <td>mem-heap-used-mb</td>
+ <td>Size of used heap memory (in megabytes)</td>
+ </tr>
+ <tr>
+ <td>mem-non-heap-commited-mb</td>
+ <td>Size of comitted non-heap memory (in megabytes)</td>
+ </tr>
+ <tr>
+ <td>mem-non-heap-max-mb</td>
+ <td>Size of maximum non-heap memory (in megabytes)</td>
+ </tr>
+ <tr>
+ <td>mem-non-heap-used-mb</td>
+ <td>Size of used non-heap memory (in megabytes)</td>
+ </tr>
+ <tr>
+ <td><span class="task">gc-name</span>-gc-count</td>
+ <td>Number of garbage collection calls (for each garbage collector)</td>
+ </tr>
+ <tr>
+ <td><span class="task">gc-name</span>-gc-time-millis</td>
+ <td>Total time spent in garbage collection (for each garbage collector) (in milliseconds)</td>
+ </tr>
+ <tr>
+ <td>threads-new</td>
+ <td>Current number of threads not started yet</td>
+ </tr>
+ <tr>
+ <td>threads-runnable</td>
+ <td>Current number of running threads</td>
+ </tr>
+ <tr>
+ <td>threads-timed-waiting</td>
+ <td>Current number of timed-waiting threads</td>
+ </tr>
+ <tr>
+ <td>threads-waiting</td>
+ <td>Current number of waiting threads</td>
+ </tr>
+ <tr>
+ <td>threads-blocked</td>
+ <td>Current number of blocked threads</td>
+ </tr>
+ <tr>
+ <td>threads-terminated</td>
+ <td>Current number of terminated threads</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="system-consumers-metrics">org.apache.samza.system.SystemConsumersMetrics</th>
+ </tr>
+ <tr>
+ <td>chose-null</td>
+ <td>Number of times message chooser returned null message envelope</td>
+ </tr>
+ <tr>
+ <td>chose-object</td>
+ <td>Number of times message chooser returned an incoming message envelope</td>
+ </tr>
+ <tr>
+ <td>deserialization error</td>
+ <td>Number of times an incoming message was not deserialized successfully</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-polls</td>
+ <td>Number of times the poll method for particular system was called</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-ssp-fetches-per-poll</td>
+ <td>Number representing how many partitions of a particular system were provided to the SystemConsumer to get new messages</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-messages-per-poll</td>
+ <td>Number of times the SystemConsumer for underlying system was actually polled to get new messages</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="stream">stream</span>-<span class="partition">partition</span>-messages-chosen</td>
+ <td>Number of messages that were chosen by the MessageChooser for particular system stream partition.</td>
+ </tr>
+ <tr>
+ <td>poll-ns</td>
+ <td><a href="#average-time">Average time</a> spent polling all underlying systems for new messages (in nanoseconds)</td>
+ </tr>
+ <tr>
+ <td>deserialization-ns</td>
+ <td><a href="#average-time">Average time</a> spent deserializing incoming messages (in nanoseconds)</td>
+ </tr>
+ <tr>
+ <td>ssps-needed-by-chooser</td>
+ <td>Current number of empty system stream partitions</td>
+ </tr>
+ <tr>
+ <td>poll-timeout</td>
+ <td>Current poll timeout</td>
+ </tr>
+ <tr>
+ <td>unprocessed-messages</td>
+ <td>Current number of unprocessed messages in buffer of incoming messages that are eligible for choosing</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="system-producers-metrics">org.apache.samza.system.SystemProducersMetrics</th>
+ </tr>
+ <tr>
+ <td>sends</td>
+ <td>Number of send method calls. Representing total number of sent messages</td>
+ </tr>
+ <tr>
+ <td>flushes</td>
+ <td>Number of flush method calls. Flush method flushes messages for all registered producers</td>
+ </tr>
+ <tr>
+ <td><span class="source">source</span>-sends</td>
+ <td>Number of sent messages for particular source (task instance)</td>
+ </tr>
+ <tr>
+ <td><span class="source">source</span>-flushes</td>
+ <td>Number of flushes for particular source (task instance)</td>
+ </tr>
+ <tr>
+ <td>serialization error</td>
+ <td>Number of errors occured while serializing envelopes before sending</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="container-process-manager-metrics">org.apache.samza.metrics.ContainerProcessManagerMetrics</th>
+ </tr>
+ <tr>
+ <td>running-containers</td>
+ <td>Number of running containers</td>
+ </tr>
+ <tr>
+ <td>needed-containers</td>
+ <td>Number of containers needed for the job to be declared healthy</td>
+ </tr>
+ <tr>
+ <td>completed-containers</td>
+ <td>Number of containers that have completed their execution and exited successfully</td>
+ </tr>
+ <tr>
+ <td>failed-containers</td>
+ <td>Number of failed containers</td>
+ </tr>
+ <tr>
+ <td>released-containers</td>
+ <td>Number of containers released due to extra allocation returned by the RM</td>
+ </tr>
+ <tr>
+ <td>container-count</td>
+ <td>Number of containers configured for the job</td>
+ </tr>
+ <tr>
+ <td>job-healthy</td>
+ <td>State indicating whether the job is healthy or not</td>
+ </tr>
+ <tr>
+ <td>locality-matched</td>
+ <td>Ratio beetwen number of container assigned to the prefered host and total number of container allocation requests</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="samza-app-master-metrics">org.apache.samza.job.yarn.SamzaAppMasterMetrics <span style="color: #f00">(Deprecated)</span><br/><span style="font-weight: normal;margin-left:40px; color : #f00">- After deprecation, all the following metrics will be under new group <a href="#container-process-manager-metrics">"org.apache.samza.metrics.ContainerProcessManagerMetrics"</a> mentioned above</span></th>
+ </tr>
+ <tr>
+ <td>running-containers</td>
+ <td>Number of running containers</td>
+ </tr>
+ <tr>
+ <td>needed-containers</td>
+ <td>Number of containers needed for the job to be declared healthy</td>
+ </tr>
+ <tr>
+ <td>completed-containers</td>
+ <td>Number of containers that have completed their execution and exited successfully</td>
+ </tr>
+ <tr>
+ <td>failed-containers</td>
+ <td>Number of failed containers</td>
+ </tr>
+ <tr>
+ <td>released-containers</td>
+ <td>Number of containers released due to extra allocation returned by the RM</td>
+ </tr>
+ <tr>
+ <td>container-count</td>
+ <td>Number of containers configured for the job</td>
+ </tr>
+ <tr>
+ <td>job-healthy</td>
+ <td>State indicating whether the job is healthy or not</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="kafka-system-consumer-metrics">org.apache.samza.system.kafka.KafkaSystemConsumerMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="topic">topic</span>-<span class="partition">partition</span>-offset-change</td>
+ <td>The next offset to be read for a topic partition</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="topic">topic</span>-<span class="partition">partition</span>-bytes-read</td>
+ <td>Total size of all messages read for a topic partition (payload + key size)</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="topic">topic</span>-<span class="partition">partition</span>-messages-read</td>
+ <td>Number of messages read for a topic partition</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="topic">topic</span>-<span class="partition">partition</span>-high-watermark</td>
+ <td>Offset of the last committed message in kafka's topic partition</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="topic">topic</span>-<span class="partition">partition</span>-messages-behind-high-watermark</td>
+ <td>Number of messages that are not fetched yet (difference beetwen high watermark and next offset)</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="host">host</span>-<span class="port">port</span>-reconnects</td>
+ <td>Number of reconnects to broker on particular host and port.</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="host">host</span>-<span class="port">port</span>-bytes-read</td>
+ <td>Total size of all messages read from a broker on particular host and port</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="host">host</span>-<span class="port">port</span>-messages-read</td>
+ <td>Number of times consumer fetched a broker on a particular host and port to get new messages</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="host">host</span>-<span class="port">port</span>-skipped-fetch-requests</td>
+ <td>Number of times fetchMessage method is called but no topic/partitions needed new messages</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="host">host</span>-<span class="port">port</span>-topic-partitions</td>
+ <td>Number of broker's topic partitions which are being consumed</td>
+ </tr>
+ <tr>
+ <td>poll-count</td>
+ <td>Number of polling the KafkaSystemConsumer to get any available messages</td>
+ </tr>
+ <tr>
+ <td>no-more-messages-SystemStreamPartition [<span class="system">system</span>, <span class="stream">stream</span>, <span class="partition">partition</span>]</td>
+ <td>Indicates if kafka consumer is at head for particular partition</td>
+ </tr>
+ <tr>
+ <td>blocking-poll-count-SystemStreamPartition [<span class="system">system</span>, <span class="stream">stream</span>, <span class="partition">partition</span>]</td>
+ <td>Number that represents how many times blocking poll is executed (polling until we get at least one message, or until we catch up to the head of the stream) (per partition)</td>
+ </tr>
+ <tr>
+ <td>blocking-poll-timeout-count-SystemStreamPartition [<span class="system">system</span>, <span class="stream">stream</span>, <span class="partition">partition</span>]</td>
+ <td>Number that represents how many times timed blocking poll is executed (polling until we get at least one message within a timeout period) (per partition)</td>
+ </tr>
+ <tr>
+ <td>buffered-message-count-SystemStreamPartition [<span class="system">system</span>, <span class="stream">stream</span>, <span class="partition">partition</span>]</td>
+ <td>Current number of messages in queue (per partition)</td>
+ </tr>
+ <tr>
+ <td>buffered-message-size-SystemStreamPartition [<span class="system">system</span>, <span class="stream">stream</span>, <span class="partition">partition</span>]</td>
+ <td>Current size of messages in queue (if systems.<span class="system">system</span>.samza.fetch.threshold.bytes is defined) (per partition)</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="kafka-system-producer-metrics">org.apache.samza.system.kafka.KafkaSystemProducerMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-producer-sends</td>
+ <td>Number of calls made to send in KafkaSystemProducer</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-producer-send-success</td>
+ <td>Number of send requests that was successfully completed by the KafkaSystemProducer</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-producer-send-failed</td>
+ <td>Number of send requests that was failed by the KafkaSystemProducer (due to unrecoverable errors)</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-producer-retries <span style="color: #f00">(Deprecated)</span></td>
+ <td>Number of times the system producer retries a send request (due to RetriableException)</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flushes</td>
+ <td>Number of calls made to flush in KafkaSystemProducer</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flush-ns</td>
+ <td>Represents <a href="#average-time">average time</a> the flush call takes to complete (in nanoseconds)
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flush-failed</td>
+ <td>Number of times flush operation failed</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="key-value-storage-engine-metrics">org.apache.samza.storage.kv.KeyValueStorageEngineMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-puts</td>
+ <td>Total number of updating the mapping of the specified key-value pair in the engine's key value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-put-ns</td>
+ <td><a href="#average-time">Average duration</a> of updating the mapping of the key-value pairs in the engine's key value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-gets</td>
+ <td>Total number of looking for the <code>value</code> associated with the specified <code>key</code> in the engine's key value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-get-ns</td>
+ <td><a href="#average-time">Average duration</a> of looking for the <code>value</code> associated with the specified <code>key</code> in the engine's key value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-alls</td>
+ <td>Total number of accessing the iterator for all entries in engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-all-ns</td>
+ <td><a href="#average-time">Average duration</a> of accessing the iterator for all entries in engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-ranges</td>
+ <td>Total number of accessing the iterator for a sorted range of entries in engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-range-ns</td>
+ <td><a href="#average-time">Average duration</a> of accessing the iterator for a sorted range of entries in engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-deletes</td>
+ <td>Total number of deleting the mapping for the specified <code>key</code> from engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-delete-ns</td>
+ <td><a href="#average-time">Average duration</a> of deleting mappings for the specifide <code>key</code> from engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-flushes</td>
+ <td>Total number of flushing engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-flush-ns</td>
+ <td><a href="#average-time">Average duration</a> of flushing engine's key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-messages-restored</td>
+ <td>Number of restored messages (key-value pairs) from the change log</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-message-bytes</td>
+ <td>Total amount of bytes (key and value size) restored from the change log</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="key-value-store-metrics">org.apache.samza.storage.kv.KeyValueStoreMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-puts</td>
+ <td>Total number of updating the mapping of the specified key-value pair. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#put-K-V-"><code>put</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-gets</td>
+ <td>Total number of looking for the <code>value</code> associated with the specified <code>key</code>. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#get-K-"><code>get</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-getalls</td>
+ <td>Total number of looking for <code>values</code> associated with the specified <code>keys</code>. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#getAll-java.util.List-"><code>getAll</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-alls</td>
+ <td>Total number of accessing the iterator for all entries in this key-value store. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#all--"><code>all</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-ranges</td>
+ <td>Total number of accessing the iterator for a sorted range of entries. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#range-K-K-"><code>range</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-bytes-written</td>
+ <td>Total amount of bytes that are written to the key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-bytes-read</td>
+ <td>Total amount of bytes that are read from the key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-flushes</td>
+ <td>Total number of key-value store flushes. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#flush--"><code>flush</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-deletes</td>
+ <td>Total number of deleting the mapping for the specified <code>key</code> from the key-value store. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#delete-K-"><code>delete</code></a> method</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-deletealls</td>
+ <td>Total number of deleting the mappings for the specified <code>keys</code> from this key-value store. Representing total number of calls to the <a href="../api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html#deleteAll-java.util.List-"><code>deleteAll</code></a> method</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="logged-store-metrics">org.apache.samza.storage.kv.LoggedStoreMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-puts</td>
+ <td>Total number of updating the mapping of the specified key-value pair in the underlying key-value store and logging it out to the changelog</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-gets</td>
+ <td>Total number of looking for the <code>value</code> associated with the specified <code>key</code> from the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-alls</td>
+ <td>Total number of accessing the iterator for all entries in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-ranges</td>
+ <td>Total number of accessing the iterator for a sorted range of entries in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-deletes</td>
+ <td>Total number of deleting the mapping for the specified <code>key</code> from the underlying key-value store and logging it out to the changelog as a null value</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-flushes</td>
+ <td>Total number of flushing the underlying key-value store</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="serialized-key-value-store-metrics">org.apache.samza.storage.kv.SerializedKeyValueStoreMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-puts</td>
+ <td>Total number of updating the mapping of the specified key-value pair in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-gets</td>
+ <td>Total number of looking for the <code>value</code> associated with the specified <code>key</code></td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-alls</td>
+ <td>Total number of accessing the iterator for all entries in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-ranges</td>
+ <td>Total number of accessing the iterator for a sorted range of entries in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-bytes-serialized</td>
+ <td>Total amount of bytes serialized while writing mappings to the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-bytes-deserialized</td>
+ <td>Total amount of bytes deserialized while reading mappings from the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-deletes</td>
+ <td>Total number of deleting the mapping for the specified <code>key</code> from the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-flushes</td>
+ <td>Total number of flushing the underlying key-value store</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="cached-store-metrics">org.apache.samza.storage.kv.CachedStoreMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-puts</td>
+ <td>Total number of updating the mapping of the specified key-value pair in the cached store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-put-all-dirty-entires-batch-size</td>
+ <td>Total number of dirty mappings that have been written to the underlying store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-gets</td>
+ <td>Total number of looking for the <code>value</code> associated with the specified <code>key</code> in the cached storage</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-alls</td>
+ <td>Total number of accessing the iterator for all entries in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-ranges</td>
+ <td>Total number of accessing the iterator for a sorted range of entries in the underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-flushes</td>
+ <td>Total number of purging dirty entries in the cache and flushing underlying key-value store</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-deletes</td>
+ <td>Total number of deleting the mapping for the specified <code>key</code> from the cache storage</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-cache-hits</td>
+ <td>Total number of times the item in the cache storage is read</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-cache-size</td>
+ <td>The number of items currently in the cache</td>
+ </tr>
+ <tr>
+ <td><span class="store">store-name</span>-dirty-count</td>
+ <td>The number of items currently in the dirty list</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="round-robin-chooser-metrics">org.apache.samza.system.chooser.RoundRobinChooserMetrics</th>
+ </tr>
+ <tr>
+ <td>buffered-messages</td>
+ <td>Size of the queue with potential messages to process</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="batching-chooser-metrics">org.apache.samza.system.chooser.BatchingChooserMetrics</th>
+ </tr>
+ <tr>
+ <td>batch-resets</td>
+ <td>The number of batch resets due to exceeding max batch size limit</td>
+ </tr>
+ <tr>
+ <td>batched-envelopes</td>
+ <td>The number of envelopes in the current batch</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="bootstrapping-chooser-metrics">org.apache.samza.system.chooser.BootstrappingChooserMetrics</th>
+ </tr>
+ <tr>
+ <td>lagging-batch-streams</td>
+ <td>The number of bootstrapping streams that are lagging</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-<span class="stream">stream</span>-lagging-partitions</td>
+ <td>The number of lagging partitions in the stream (for each stream marked as bootstrapping stream)</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="hdfs-system-producer-metrics">org.apache.samza.system.hdfs.HdfsSystemProducerMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-producer-sends</td>
+ <td>Total number of attempts to write to the HDFS</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-send-success</td>
+ <td>Total number of successful writes to the HDFS</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-send-failed</td>
+ <td>Total number of failures while sending envelopes to the HDFS</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-send-ms</td>
+ <td><a href="#average-time">Average time</a> spent for writing messages to the HDFS (in milliseconds)</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flushes</td>
+ <td>Total number of attempts to flush data to the file system</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flush-success</td>
+ <td>Total number of successfully flushed all written data to the file system</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flush-failed</td>
+ <td>Total number of failures while flushing data to the file system</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-flush-ms</td>
+ <td><a href="#average-time">Average time</a> spent for flushing messages to the HDFS (in milliseconds)</td>
+ </tr>
+
+ <tr>
+ <th colspan="2" class="section" id="elasticsearch-system-producer-metrics">org.apache.samza.system.elasticsearch.ElasticsearchSystemProducerMetrics</th>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-bulk-send-success</td>
+ <td>Total number of successfully sent bulk requests</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-docs-inserted</td>
+ <td>Total number of created documents</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-docs-updated</td>
+ <td>Total number of document updates</td>
+ </tr>
+ <tr>
+ <td><span class="system">system</span>-version-conflicts</td>
+ <td>Number of times the request could not be completed due to a conflict with the current state of the document</td>
+ </tr>
+ </tbody>
+</table>
+</body>
+</html>
\ No newline at end of file