You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by xi...@apache.org on 2018/05/25 22:28:41 UTC
svn commit: r1832281 [8/15] - in /samza/site: ./ community/ contribute/
learn/documentation/latest/ learn/documentation/latest/api/
learn/documentation/latest/api/javadocs/
learn/documentation/latest/api/javadocs/org/apache/samza/
learn/documentation/l...
Modified: samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/NoOpMetricsRegistry.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/NoOpMetricsRegistry.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/NoOpMetricsRegistry.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/NoOpMetricsRegistry.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:52 PST 2018 -->
-<title>NoOpMetricsRegistry (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:06 PDT 2018 -->
+<title>NoOpMetricsRegistry (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="NoOpMetricsRegistry (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="NoOpMetricsRegistry (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -49,7 +49,7 @@ var activeTableTab = "activeTableTab";
<div class="subNav">
<ul class="navList">
<li><a href="../../../../org/apache/samza/util/Clock.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
-<li><a href="../../../../org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html" title="class in org.apache.samza.util"><span class="typeNameLink">Next Class</span></a></li>
+<li><a href="../../../../org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Next Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../index.html?org/apache/samza/util/NoOpMetricsRegistry.html" target="_top">Frames</a></li>
@@ -396,7 +396,7 @@ implements <a href="../../../../org/apac
<div class="subNav">
<ul class="navList">
<li><a href="../../../../org/apache/samza/util/Clock.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
-<li><a href="../../../../org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html" title="class in org.apache.samza.util"><span class="typeNameLink">Next Class</span></a></li>
+<li><a href="../../../../org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Next Class</span></a></li>
</ul>
<ul class="navList">
<li><a href="../../../../index.html?org/apache/samza/util/NoOpMetricsRegistry.html" target="_top">Frames</a></li>
Added: samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/RateLimiter.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/RateLimiter.html?rev=1832281&view=auto
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/RateLimiter.html (added)
+++ samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/RateLimiter.html Fri May 25 22:28:38 2018
@@ -0,0 +1,380 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<!-- NewPage -->
+<html lang="en">
+<head>
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:06 PDT 2018 -->
+<title>RateLimiter (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
+<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
+<script type="text/javascript" src="../../../../script.js"></script>
+</head>
+<body>
+<script type="text/javascript"><!--
+ try {
+ if (location.href.indexOf('is-external=true') == -1) {
+ parent.document.title="RateLimiter (samza-api 0.14.1 API)";
+ }
+ }
+ catch(err) {
+ }
+//-->
+var methods = {"i0":6,"i1":6,"i2":6,"i3":6,"i4":6,"i5":6};
+var tabs = {65535:["t0","All Methods"],2:["t2","Instance Methods"],4:["t3","Abstract Methods"]};
+var altColor = "altColor";
+var rowColor = "rowColor";
+var tableTab = "tableTab";
+var activeTableTab = "activeTableTab";
+</script>
+<noscript>
+<div>JavaScript is disabled on your browser.</div>
+</noscript>
+<!-- ========= START OF TOP NAVBAR ======= -->
+<div class="topNav"><a name="navbar.top">
+<!-- -->
+</a>
+<div class="skipNav"><a href="#skip.navbar.top" title="Skip navigation links">Skip navigation links</a></div>
+<a name="navbar.top.firstrow">
+<!-- -->
+</a>
+<ul class="navList" title="Navigation">
+<li><a href="../../../../overview-summary.html">Overview</a></li>
+<li><a href="package-summary.html">Package</a></li>
+<li class="navBarCell1Rev">Class</li>
+<li><a href="package-tree.html">Tree</a></li>
+<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
+<li><a href="../../../../index-all.html">Index</a></li>
+<li><a href="../../../../help-doc.html">Help</a></li>
+</ul>
+</div>
+<div class="subNav">
+<ul class="navList">
+<li><a href="../../../../org/apache/samza/util/NoOpMetricsRegistry.html" title="class in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
+<li><a href="../../../../org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html" title="class in org.apache.samza.util"><span class="typeNameLink">Next Class</span></a></li>
+</ul>
+<ul class="navList">
+<li><a href="../../../../index.html?org/apache/samza/util/RateLimiter.html" target="_top">Frames</a></li>
+<li><a href="RateLimiter.html" target="_top">No Frames</a></li>
+</ul>
+<ul class="navList" id="allclasses_navbar_top">
+<li><a href="../../../../allclasses-noframe.html">All Classes</a></li>
+</ul>
+<div>
+<script type="text/javascript"><!--
+ allClassesLink = document.getElementById("allclasses_navbar_top");
+ if(window==top) {
+ allClassesLink.style.display = "block";
+ }
+ else {
+ allClassesLink.style.display = "none";
+ }
+ //-->
+</script>
+</div>
+<div>
+<ul class="subNavList">
+<li>Summary: </li>
+<li>Nested | </li>
+<li>Field | </li>
+<li>Constr | </li>
+<li><a href="#method.summary">Method</a></li>
+</ul>
+<ul class="subNavList">
+<li>Detail: </li>
+<li>Field | </li>
+<li>Constr | </li>
+<li><a href="#method.detail">Method</a></li>
+</ul>
+</div>
+<a name="skip.navbar.top">
+<!-- -->
+</a></div>
+<!-- ========= END OF TOP NAVBAR ========= -->
+<!-- ======== START OF CLASS DATA ======== -->
+<div class="header">
+<div class="subTitle">org.apache.samza.util</div>
+<h2 title="Interface RateLimiter" class="title">Interface RateLimiter</h2>
+</div>
+<div class="contentContainer">
+<div class="description">
+<ul class="blockList">
+<li class="blockList">
+<dl>
+<dt>All Superinterfaces:</dt>
+<dd>java.io.Serializable</dd>
+</dl>
+<hr>
+<br>
+<pre><a href="../../../../org/apache/samza/annotation/InterfaceStability.Unstable.html" title="annotation in org.apache.samza.annotation">@InterfaceStability.Unstable</a>
+public interface <span class="typeNameLabel">RateLimiter</span>
+extends java.io.Serializable</pre>
+<div class="block">A rate limiter interface used by Samza components to limit throughput of operations
+ against a resource. Operations against a resource are represented by credits.
+ Resources could be streams, databases, web services, etc.
+
+ <p>
+ This interface supports two categories of policies: tagged and non-tagged.
+ Tagged rate limiter is used, when further differentiation is required within a resource.
+ For example: messages in a stream may be treated differently depending on the
+ overall situation of processing; or read/write operations to a database.
+ Tagging is the mechanism to allow this differentiation.
+
+ <p>
+ The following types of invocations are provided
+ <ul>
+ <li>Block indefinitely until requested credits become available</li>
+ <li>Block for a provided amount of time, then return available credits</li>
+ </ul></div>
+</li>
+</ul>
+</div>
+<div class="summary">
+<ul class="blockList">
+<li class="blockList">
+<!-- ========== METHOD SUMMARY =========== -->
+<ul class="blockList">
+<li class="blockList"><a name="method.summary">
+<!-- -->
+</a>
+<h3>Method Summary</h3>
+<table class="memberSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation">
+<caption><span id="t0" class="activeTableTab"><span>All Methods</span><span class="tabEnd"> </span></span><span id="t2" class="tableTab"><span><a href="javascript:show(2);">Instance Methods</a></span><span class="tabEnd"> </span></span><span id="t3" class="tableTab"><span><a href="javascript:show(4);">Abstract Methods</a></span><span class="tabEnd"> </span></span></caption>
+<tr>
+<th class="colFirst" scope="col">Modifier and Type</th>
+<th class="colLast" scope="col">Method and Description</th>
+</tr>
+<tr id="i0" class="altColor">
+<td class="colFirst"><code>void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../org/apache/samza/util/RateLimiter.html#acquire-int-">acquire</a></span>(int numberOfCredit)</code>
+<div class="block">Attempt to acquire the provided number of credits, blocks indefinitely until
+ all requested credits become available.</div>
+</td>
+</tr>
+<tr id="i1" class="rowColor">
+<td class="colFirst"><code>int</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../org/apache/samza/util/RateLimiter.html#acquire-int-long-java.util.concurrent.TimeUnit-">acquire</a></span>(int numberOfCredit,
+ long timeout,
+ java.util.concurrent.TimeUnit unit)</code>
+<div class="block">Attempt to acquire the provided number of credits, blocks for up to provided amount of
+ time for credits to become available.</div>
+</td>
+</tr>
+<tr id="i2" class="altColor">
+<td class="colFirst"><code>void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../org/apache/samza/util/RateLimiter.html#acquire-java.util.Map-">acquire</a></span>(java.util.Map<java.lang.String,java.lang.Integer> tagToCreditMap)</code>
+<div class="block">Attempt to acquire the provided number of credits for a number of tags, blocks indefinitely
+ until all requested credits become available</div>
+</td>
+</tr>
+<tr id="i3" class="rowColor">
+<td class="colFirst"><code>java.util.Map<java.lang.String,java.lang.Integer></code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../org/apache/samza/util/RateLimiter.html#acquire-java.util.Map-long-java.util.concurrent.TimeUnit-">acquire</a></span>(java.util.Map<java.lang.String,java.lang.Integer> tagToCreditMap,
+ long timeout,
+ java.util.concurrent.TimeUnit unit)</code>
+<div class="block">Attempt to acquire the provided number of credits for a number of tags, blocks for up to provided amount of
+ time for credits to become available.</div>
+</td>
+</tr>
+<tr id="i4" class="altColor">
+<td class="colFirst"><code>java.util.Set<java.lang.String></code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../org/apache/samza/util/RateLimiter.html#getSupportedTags--">getSupportedTags</a></span>()</code>
+<div class="block">Get the entire set of tags for which we have configured credits for rate limiting.</div>
+</td>
+</tr>
+<tr id="i5" class="rowColor">
+<td class="colFirst"><code>void</code></td>
+<td class="colLast"><code><span class="memberNameLink"><a href="../../../../org/apache/samza/util/RateLimiter.html#init-org.apache.samza.config.Config-org.apache.samza.task.TaskContext-">init</a></span>(<a href="../../../../org/apache/samza/config/Config.html" title="class in org.apache.samza.config">Config</a> config,
+ <a href="../../../../org/apache/samza/task/TaskContext.html" title="interface in org.apache.samza.task">TaskContext</a> taskContext)</code>
+<div class="block">Initialize this rate limiter, this method should be called during container initialization.</div>
+</td>
+</tr>
+</table>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+<div class="details">
+<ul class="blockList">
+<li class="blockList">
+<!-- ============ METHOD DETAIL ========== -->
+<ul class="blockList">
+<li class="blockList"><a name="method.detail">
+<!-- -->
+</a>
+<h3>Method Detail</h3>
+<a name="init-org.apache.samza.config.Config-org.apache.samza.task.TaskContext-">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>init</h4>
+<pre>void init(<a href="../../../../org/apache/samza/config/Config.html" title="class in org.apache.samza.config">Config</a> config,
+ <a href="../../../../org/apache/samza/task/TaskContext.html" title="interface in org.apache.samza.task">TaskContext</a> taskContext)</pre>
+<div class="block">Initialize this rate limiter, this method should be called during container initialization.</div>
+<dl>
+<dt><span class="paramLabel">Parameters:</span></dt>
+<dd><code>config</code> - job configuration</dd>
+<dd><code>taskContext</code> - task context that owns this rate limiter</dd>
+</dl>
+</li>
+</ul>
+<a name="acquire-int-">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>acquire</h4>
+<pre>void acquire(int numberOfCredit)</pre>
+<div class="block">Attempt to acquire the provided number of credits, blocks indefinitely until
+ all requested credits become available.</div>
+<dl>
+<dt><span class="paramLabel">Parameters:</span></dt>
+<dd><code>numberOfCredit</code> - requested number of credits</dd>
+</dl>
+</li>
+</ul>
+<a name="acquire-int-long-java.util.concurrent.TimeUnit-">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>acquire</h4>
+<pre>int acquire(int numberOfCredit,
+ long timeout,
+ java.util.concurrent.TimeUnit unit)</pre>
+<div class="block">Attempt to acquire the provided number of credits, blocks for up to provided amount of
+ time for credits to become available. When timeout elapses and not all required credits
+ can be acquired, it returns the number of credits currently available. It may return
+ immediately, if it determines no credits can be acquired during the provided amount time.</div>
+<dl>
+<dt><span class="paramLabel">Parameters:</span></dt>
+<dd><code>numberOfCredit</code> - requested number of credits</dd>
+<dd><code>timeout</code> - number of time unit to wait</dd>
+<dd><code>unit</code> - time unit to for timeout</dd>
+<dt><span class="returnLabel">Returns:</span></dt>
+<dd>number of credits acquired</dd>
+</dl>
+</li>
+</ul>
+<a name="acquire-java.util.Map-">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>acquire</h4>
+<pre>void acquire(java.util.Map<java.lang.String,java.lang.Integer> tagToCreditMap)</pre>
+<div class="block">Attempt to acquire the provided number of credits for a number of tags, blocks indefinitely
+ until all requested credits become available</div>
+<dl>
+<dt><span class="paramLabel">Parameters:</span></dt>
+<dd><code>tagToCreditMap</code> - a map of requested number of credits keyed by tag</dd>
+</dl>
+</li>
+</ul>
+<a name="acquire-java.util.Map-long-java.util.concurrent.TimeUnit-">
+<!-- -->
+</a>
+<ul class="blockList">
+<li class="blockList">
+<h4>acquire</h4>
+<pre>java.util.Map<java.lang.String,java.lang.Integer> acquire(java.util.Map<java.lang.String,java.lang.Integer> tagToCreditMap,
+ long timeout,
+ java.util.concurrent.TimeUnit unit)</pre>
+<div class="block">Attempt to acquire the provided number of credits for a number of tags, blocks for up to provided amount of
+ time for credits to become available. When timeout elapses and not all required credits
+ can be acquired, it returns the number of credits currently available. It may return
+ immediately, if it determines no credits can be acquired during the provided amount time.</div>
+<dl>
+<dt><span class="paramLabel">Parameters:</span></dt>
+<dd><code>tagToCreditMap</code> - a map of requested number of credits keyed by tag</dd>
+<dd><code>timeout</code> - number of time unit to wait</dd>
+<dd><code>unit</code> - time unit to for timeout</dd>
+<dt><span class="returnLabel">Returns:</span></dt>
+<dd>a map of number of credits acquired keyed by tag</dd>
+</dl>
+</li>
+</ul>
+<a name="getSupportedTags--">
+<!-- -->
+</a>
+<ul class="blockListLast">
+<li class="blockList">
+<h4>getSupportedTags</h4>
+<pre>java.util.Set<java.lang.String> getSupportedTags()</pre>
+<div class="block">Get the entire set of tags for which we have configured credits for rate limiting.</div>
+<dl>
+<dt><span class="returnLabel">Returns:</span></dt>
+<dd>set of supported tags</dd>
+</dl>
+</li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+</div>
+<!-- ========= END OF CLASS DATA ========= -->
+<!-- ======= START OF BOTTOM NAVBAR ====== -->
+<div class="bottomNav"><a name="navbar.bottom">
+<!-- -->
+</a>
+<div class="skipNav"><a href="#skip.navbar.bottom" title="Skip navigation links">Skip navigation links</a></div>
+<a name="navbar.bottom.firstrow">
+<!-- -->
+</a>
+<ul class="navList" title="Navigation">
+<li><a href="../../../../overview-summary.html">Overview</a></li>
+<li><a href="package-summary.html">Package</a></li>
+<li class="navBarCell1Rev">Class</li>
+<li><a href="package-tree.html">Tree</a></li>
+<li><a href="../../../../deprecated-list.html">Deprecated</a></li>
+<li><a href="../../../../index-all.html">Index</a></li>
+<li><a href="../../../../help-doc.html">Help</a></li>
+</ul>
+</div>
+<div class="subNav">
+<ul class="navList">
+<li><a href="../../../../org/apache/samza/util/NoOpMetricsRegistry.html" title="class in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
+<li><a href="../../../../org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html" title="class in org.apache.samza.util"><span class="typeNameLink">Next Class</span></a></li>
+</ul>
+<ul class="navList">
+<li><a href="../../../../index.html?org/apache/samza/util/RateLimiter.html" target="_top">Frames</a></li>
+<li><a href="RateLimiter.html" target="_top">No Frames</a></li>
+</ul>
+<ul class="navList" id="allclasses_navbar_bottom">
+<li><a href="../../../../allclasses-noframe.html">All Classes</a></li>
+</ul>
+<div>
+<script type="text/javascript"><!--
+ allClassesLink = document.getElementById("allclasses_navbar_bottom");
+ if(window==top) {
+ allClassesLink.style.display = "block";
+ }
+ else {
+ allClassesLink.style.display = "none";
+ }
+ //-->
+</script>
+</div>
+<div>
+<ul class="subNavList">
+<li>Summary: </li>
+<li>Nested | </li>
+<li>Field | </li>
+<li>Constr | </li>
+<li><a href="#method.summary">Method</a></li>
+</ul>
+<ul class="subNavList">
+<li>Detail: </li>
+<li>Field | </li>
+<li>Constr | </li>
+<li><a href="#method.detail">Method</a></li>
+</ul>
+</div>
+<a name="skip.navbar.bottom">
+<!-- -->
+</a></div>
+<!-- ======== END OF BOTTOM NAVBAR ======= -->
+</body>
+</html>
Modified: samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/SinglePartitionWithoutOffsetsSystemAdmin.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:52 PST 2018 -->
-<title>SinglePartitionWithoutOffsetsSystemAdmin (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:06 PDT 2018 -->
+<title>SinglePartitionWithoutOffsetsSystemAdmin (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="SinglePartitionWithoutOffsetsSystemAdmin (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="SinglePartitionWithoutOffsetsSystemAdmin (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -48,7 +48,7 @@ var activeTableTab = "activeTableTab";
</div>
<div class="subNav">
<ul class="navList">
-<li><a href="../../../../org/apache/samza/util/NoOpMetricsRegistry.html" title="class in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
+<li><a href="../../../../org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
<li>Next Class</li>
</ul>
<ul class="navList">
@@ -188,7 +188,7 @@ implements <a href="../../../../org/apac
<!-- -->
</a>
<h3>Methods inherited from interface org.apache.samza.system.<a href="../../../../org/apache/samza/system/SystemAdmin.html" title="interface in org.apache.samza.system">SystemAdmin</a></h3>
-<code><a href="../../../../org/apache/samza/system/SystemAdmin.html#clearStream-org.apache.samza.system.StreamSpec-">clearStream</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#createStream-org.apache.samza.system.StreamSpec-">createStream</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#validateStream-org.apache.samza.system.StreamSpec-">validateStream</a></code></li>
+<code><a href="../../../../org/apache/samza/system/SystemAdmin.html#clearStream-org.apache.samza.system.StreamSpec-">clearStream</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#createStream-org.apache.samza.system.StreamSpec-">createStream</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#deleteMessages-java.util.Map-">deleteMessages</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#start--">start</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#stop--">stop</a>, <a href="../../../../org/apache/samza/system/SystemAdmin.html#validateStream-org.apache.samza.system.StreamSpec-">validateStream</a></code></li>
</ul>
</li>
</ul>
@@ -312,7 +312,7 @@ implements <a href="../../../../org/apac
</div>
<div class="subNav">
<ul class="navList">
-<li><a href="../../../../org/apache/samza/util/NoOpMetricsRegistry.html" title="class in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
+<li><a href="../../../../org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Prev Class</span></a></li>
<li>Next Class</li>
</ul>
<ul class="navList">
Modified: samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-frame.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-frame.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-frame.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-frame.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>org.apache.samza.util (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>org.apache.samza.util (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
</head>
@@ -14,6 +14,7 @@
<h2 title="Interfaces">Interfaces</h2>
<ul title="Interfaces">
<li><a href="Clock.html" title="interface in org.apache.samza.util" target="classFrame"><span class="interfaceName">Clock</span></a></li>
+<li><a href="RateLimiter.html" title="interface in org.apache.samza.util" target="classFrame"><span class="interfaceName">RateLimiter</span></a></li>
</ul>
<h2 title="Classes">Classes</h2>
<ul title="Classes">
Modified: samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-summary.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-summary.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-summary.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-summary.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>org.apache.samza.util (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>org.apache.samza.util (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="org.apache.samza.util (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="org.apache.samza.util (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -87,6 +87,13 @@
<div class="block">Mockable interface for tracking time.</div>
</td>
</tr>
+<tr class="rowColor">
+<td class="colFirst"><a href="../../../../org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util">RateLimiter</a></td>
+<td class="colLast">
+<div class="block">A rate limiter interface used by Samza components to limit throughput of operations
+ against a resource.</div>
+</td>
+</tr>
</tbody>
</table>
</li>
Modified: samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-tree.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-tree.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-tree.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/org/apache/samza/util/package-tree.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>org.apache.samza.util Class Hierarchy (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>org.apache.samza.util Class Hierarchy (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="../../../../stylesheet.css" title="Style">
<script type="text/javascript" src="../../../../script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="org.apache.samza.util Class Hierarchy (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="org.apache.samza.util Class Hierarchy (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -96,6 +96,11 @@
<h2 title="Interface Hierarchy">Interface Hierarchy</h2>
<ul>
<li type="circle">org.apache.samza.util.<a href="../../../../org/apache/samza/util/Clock.html" title="interface in org.apache.samza.util"><span class="typeNameLink">Clock</span></a></li>
+<li type="circle">java.io.Serializable
+<ul>
+<li type="circle">org.apache.samza.util.<a href="../../../../org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util"><span class="typeNameLink">RateLimiter</span></a></li>
+</ul>
+</li>
</ul>
</div>
<!-- ======= START OF BOTTOM NAVBAR ====== -->
Modified: samza/site/learn/documentation/latest/api/javadocs/overview-frame.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/overview-frame.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/overview-frame.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/overview-frame.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>Overview List (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>Overview List (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="stylesheet.css" title="Style">
<script type="text/javascript" src="script.js"></script>
</head>
Modified: samza/site/learn/documentation/latest/api/javadocs/overview-summary.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/overview-summary.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/overview-summary.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/overview-summary.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>Overview (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>Overview (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="stylesheet.css" title="Style">
<script type="text/javascript" src="script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="Overview (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="Overview (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -69,7 +69,7 @@
</a></div>
<!-- ========= END OF TOP NAVBAR ========= -->
<div class="header">
-<h1 class="title">samza-api 0.14.1-SNAPSHOT API</h1>
+<h1 class="title">samza-api 0.14.1 API</h1>
</div>
<div class="contentContainer">
<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Packages table, listing packages, and an explanation">
Modified: samza/site/learn/documentation/latest/api/javadocs/overview-tree.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/overview-tree.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/overview-tree.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/overview-tree.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>Class Hierarchy (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>Class Hierarchy (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="stylesheet.css" title="Style">
<script type="text/javascript" src="script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="Class Hierarchy (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="Class Hierarchy (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -255,6 +255,7 @@
<li type="circle">org.apache.samza.storage.kv.<a href="org/apache/samza/storage/kv/KeyValueIterator.html" title="interface in org.apache.samza.storage.kv"><span class="typeNameLink">KeyValueIterator</span></a><K,V></li>
</ul>
</li>
+<li type="circle">org.apache.samza.storage.kv.<a href="org/apache/samza/storage/kv/KeyValueSnapshot.html" title="interface in org.apache.samza.storage.kv"><span class="typeNameLink">KeyValueSnapshot</span></a><K,V></li>
<li type="circle">org.apache.samza.storage.kv.<a href="org/apache/samza/storage/kv/KeyValueStore.html" title="interface in org.apache.samza.storage.kv"><span class="typeNameLink">KeyValueStore</span></a><K,V></li>
<li type="circle">org.apache.samza.system.chooser.<a href="org/apache/samza/system/chooser/MessageChooser.html" title="interface in org.apache.samza.system.chooser"><span class="typeNameLink">MessageChooser</span></a></li>
<li type="circle">org.apache.samza.system.chooser.<a href="org/apache/samza/system/chooser/MessageChooserFactory.html" title="interface in org.apache.samza.system.chooser"><span class="typeNameLink">MessageChooserFactory</span></a></li>
@@ -274,10 +275,11 @@
<li type="circle">org.apache.samza.runtime.<a href="org/apache/samza/runtime/ProcessorIdGenerator.html" title="interface in org.apache.samza.runtime"><span class="typeNameLink">ProcessorIdGenerator</span></a></li>
<li type="circle">org.apache.samza.metrics.<a href="org/apache/samza/metrics/ReadableMetricsRegistryListener.html" title="interface in org.apache.samza.metrics"><span class="typeNameLink">ReadableMetricsRegistryListener</span></a></li>
<li type="circle">org.apache.samza.metrics.<a href="org/apache/samza/metrics/Reservoir.html" title="interface in org.apache.samza.metrics"><span class="typeNameLink">Reservoir</span></a></li>
-<li type="circle">org.apache.samza.sql.udfs.<a href="org/apache/samza/sql/udfs/ScalarUdf.html" title="interface in org.apache.samza.sql.udfs"><span class="typeNameLink">ScalarUdf</span></a></li>
+<li type="circle">org.apache.samza.sql.udfs.<a href="org/apache/samza/sql/udfs/ScalarUdf.html" title="interface in org.apache.samza.sql.udfs"><span class="typeNameLink">ScalarUdf</span></a><T></li>
<li type="circle">org.apache.samza.serializers.<a href="org/apache/samza/serializers/SerdeFactory.html" title="interface in org.apache.samza.serializers"><span class="typeNameLink">SerdeFactory</span></a><T></li>
<li type="circle">java.io.Serializable
<ul>
+<li type="circle">org.apache.samza.util.<a href="org/apache/samza/util/RateLimiter.html" title="interface in org.apache.samza.util"><span class="typeNameLink">RateLimiter</span></a></li>
<li type="circle">org.apache.samza.serializers.<a href="org/apache/samza/serializers/Serde.html" title="interface in org.apache.samza.serializers"><span class="typeNameLink">Serde</span></a><T> (also extends org.apache.samza.serializers.<a href="org/apache/samza/serializers/Deserializer.html" title="interface in org.apache.samza.serializers">Deserializer</a><T>, org.apache.samza.serializers.<a href="org/apache/samza/serializers/Serializer.html" title="interface in org.apache.samza.serializers">Serializer</a><T>)</li>
</ul>
</li>
@@ -316,15 +318,14 @@
</ul>
</li>
<li type="circle">org.apache.samza.operators.<a href="org/apache/samza/operators/TableDescriptor.html" title="interface in org.apache.samza.operators"><span class="typeNameLink">TableDescriptor</span></a><K,V,D></li>
-<li type="circle">org.apache.samza.table.<a href="org/apache/samza/table/TableProvider.html" title="interface in org.apache.samza.table"><span class="typeNameLink">TableProvider</span></a>
-<ul>
-<li type="circle">org.apache.samza.table.<a href="org/apache/samza/table/LocalStoreBackedTableProvider.html" title="interface in org.apache.samza.table"><span class="typeNameLink">LocalStoreBackedTableProvider</span></a></li>
-</ul>
-</li>
+<li type="circle">org.apache.samza.table.<a href="org/apache/samza/table/TableProvider.html" title="interface in org.apache.samza.table"><span class="typeNameLink">TableProvider</span></a></li>
<li type="circle">org.apache.samza.table.<a href="org/apache/samza/table/TableProviderFactory.html" title="interface in org.apache.samza.table"><span class="typeNameLink">TableProviderFactory</span></a></li>
<li type="circle">org.apache.samza.task.<a href="org/apache/samza/task/TaskCallback.html" title="interface in org.apache.samza.task"><span class="typeNameLink">TaskCallback</span></a></li>
<li type="circle">org.apache.samza.task.<a href="org/apache/samza/task/TaskContext.html" title="interface in org.apache.samza.task"><span class="typeNameLink">TaskContext</span></a></li>
<li type="circle">org.apache.samza.task.<a href="org/apache/samza/task/TaskCoordinator.html" title="interface in org.apache.samza.task"><span class="typeNameLink">TaskCoordinator</span></a></li>
+<li type="circle">org.apache.samza.task.<a href="org/apache/samza/task/TimerCallback.html" title="interface in org.apache.samza.task"><span class="typeNameLink">TimerCallback</span></a><K></li>
+<li type="circle">org.apache.samza.operators.functions.<a href="org/apache/samza/operators/functions/TimerFunction.html" title="interface in org.apache.samza.operators.functions"><span class="typeNameLink">TimerFunction</span></a><K,OM></li>
+<li type="circle">org.apache.samza.operators.<a href="org/apache/samza/operators/TimerRegistry.html" title="interface in org.apache.samza.operators"><span class="typeNameLink">TimerRegistry</span></a><K></li>
<li type="circle">org.apache.samza.operators.triggers.<a href="org/apache/samza/operators/triggers/Trigger.html" title="interface in org.apache.samza.operators.triggers"><span class="typeNameLink">Trigger</span></a><M>
<ul>
<li type="circle">org.apache.samza.operators.triggers.<a href="org/apache/samza/operators/triggers/TimeBasedTrigger.html" title="interface in org.apache.samza.operators.triggers"><span class="typeNameLink">TimeBasedTrigger</span></a><M></li>
Modified: samza/site/learn/documentation/latest/api/javadocs/serialized-form.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/javadocs/serialized-form.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/javadocs/serialized-form.html (original)
+++ samza/site/learn/documentation/latest/api/javadocs/serialized-form.html Fri May 25 22:28:38 2018
@@ -2,9 +2,9 @@
<!-- NewPage -->
<html lang="en">
<head>
-<!-- Generated by javadoc (1.8.0_121) on Thu Jan 04 10:02:53 PST 2018 -->
-<title>Serialized Form (samza-api 0.14.1-SNAPSHOT API)</title>
-<meta name="date" content="2018-01-04">
+<!-- Generated by javadoc (1.8.0_121) on Thu May 24 12:04:07 PDT 2018 -->
+<title>Serialized Form (samza-api 0.14.1 API)</title>
+<meta name="date" content="2018-05-24">
<link rel="stylesheet" type="text/css" href="stylesheet.css" title="Style">
<script type="text/javascript" src="script.js"></script>
</head>
@@ -12,7 +12,7 @@
<script type="text/javascript"><!--
try {
if (location.href.indexOf('is-external=true') == -1) {
- parent.document.title="Serialized Form (samza-api 0.14.1-SNAPSHOT API)";
+ parent.document.title="Serialized Form (samza-api 0.14.1 API)";
}
}
catch(err) {
@@ -247,6 +247,9 @@
</li>
</ul>
</li>
+<li class="blockList">
+<h2 title="Package">Package org.apache.samza.util</h2>
+</li>
</ul>
</div>
<!-- ======= START OF BOTTOM NAVBAR ====== -->
Modified: samza/site/learn/documentation/latest/api/overview.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/api/overview.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/api/overview.html (original)
+++ samza/site/learn/documentation/latest/api/overview.html Fri May 25 22:28:38 2018
@@ -44,9 +44,7 @@
<!-- this icon only shows in versioned pages -->
-
-
- <a href="http://samza.apache.org/learn/documentation/0.14/api/overview.html"><i id="switch-version-button"></i></a>
+ <a href="http://samza.apache.org/learn/documentation/0.14/api/overview"><i id="switch-version-button"></i></a>
<!-- links for the navigation bar -->
@@ -149,7 +147,7 @@
<p>When writing a stream processor for Samza, you must implement either <a href="javadocs/org/apache/samza/task/StreamTask.html">StreamTask</a> or <a href="javadocs/org/apache/samza/task/AsyncStreamTask.html">AsyncStreamTask</a> interface. You should implement StreamTask for synchronous process, where the message processing is complete after the <em>process</em> method returns. An example of StreamTask is a computation that does not involve remote calls:</p>
-<div class="highlight"><pre><code class="java"><span class="kn">package</span> <span class="n">com</span><span class="o">.</span><span class="na">example</span><span class="o">.</span><span class="na">samza</span><span class="o">;</span>
+<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="kn">package</span> <span class="nn">com.example.samza</span><span class="o">;</span>
<span class="kd">public</span> <span class="kd">class</span> <span class="nc">MyTaskClass</span> <span class="kd">implements</span> <span class="n">StreamTask</span> <span class="o">{</span>
@@ -158,11 +156,11 @@
<span class="n">TaskCoordinator</span> <span class="n">coordinator</span><span class="o">)</span> <span class="o">{</span>
<span class="c1">// process message</span>
<span class="o">}</span>
-<span class="o">}</span></code></pre></div>
+<span class="o">}</span></code></pre></figure>
<p>The AsyncSteamTask interface, on the other hand, supports asynchronous process, where the message processing may not be complete after the <em>processAsync</em> method returns. Various concurrent libraries like Java NIO, ParSeq and Akka can be used here to make asynchronous calls, and the completion is marked by invoking the <a href="javadocs/org/apache/samza/task/TaskCallback.html">TaskCallback</a>. Samza will continue to process next message or shut down the container based on the callback status. An example of AsyncStreamTask is a computation that make remote calls but don’t block on the call completion:</p>
-<div class="highlight"><pre><code class="java"><span class="kn">package</span> <span class="n">com</span><span class="o">.</span><span class="na">example</span><span class="o">.</span><span class="na">samza</span><span class="o">;</span>
+<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="kn">package</span> <span class="nn">com.example.samza</span><span class="o">;</span>
<span class="kd">public</span> <span class="kd">class</span> <span class="nc">MyAsyncTaskClass</span> <span class="kd">implements</span> <span class="n">AsyncStreamTask</span> <span class="o">{</span>
@@ -173,13 +171,13 @@
<span class="c1">// process message with asynchronous calls</span>
<span class="c1">// fire callback upon completion, e.g. invoking callback from asynchronous call completion thread</span>
<span class="o">}</span>
-<span class="o">}</span></code></pre></div>
+<span class="o">}</span></code></pre></figure>
<p>When you run your job, Samza will create several instances of your class (potentially on multiple machines). These task instances process the messages in the input streams.</p>
<p>In your job’s configuration you can tell Samza which streams you want to consume. An incomplete example could look like this (see the <a href="../jobs/configuration.html">configuration documentation</a> for more detail):</p>
-<div class="highlight"><pre><code class="jproperties"><span class="c"># This is the class above, which Samza will instantiate when the job is run</span>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="c"># This is the class above, which Samza will instantiate when the job is run</span>
<span class="na">task.class</span><span class="o">=</span><span class="s">com.example.samza.MyTaskClass</span>
<span class="c"># Define a system called "kafka" (you can give it any name, and you can define</span>
@@ -193,11 +191,11 @@
<span class="na">serializers.registry.json.class</span><span class="o">=</span><span class="s">org.apache.samza.serializers.JsonSerdeFactory</span>
<span class="c"># Use the "json" serializer for messages in the "PageViewEvent" topic</span>
-<span class="na">systems.kafka.streams.PageViewEvent.samza.msg.serde</span><span class="o">=</span><span class="s">json</span></code></pre></div>
+<span class="na">systems.kafka.streams.PageViewEvent.samza.msg.serde</span><span class="o">=</span><span class="s">json</span></code></pre></figure>
<p>For each message that Samza receives from the task’s input streams, the <em>process</em> method is called. The <a href="javadocs/org/apache/samza/system/IncomingMessageEnvelope.html">envelope</a> contains three things of importance: the message, the key, and the stream that the message came from.</p>
-<div class="highlight"><pre><code class="java"><span class="cm">/** Every message that is delivered to a StreamTask is wrapped</span>
+<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="cm">/** Every message that is delivered to a StreamTask is wrapped</span>
<span class="cm"> * in an IncomingMessageEnvelope, which contains metadata about</span>
<span class="cm"> * the origin of the message. */</span>
<span class="kd">public</span> <span class="kd">class</span> <span class="nc">IncomingMessageEnvelope</span> <span class="o">{</span>
@@ -209,7 +207,7 @@
<span class="cm">/** The stream and partition that this message came from. */</span>
<span class="n">SystemStreamPartition</span> <span class="nf">getSystemStreamPartition</span><span class="o">()</span> <span class="o">{</span> <span class="o">...</span> <span class="o">}</span>
-<span class="o">}</span></code></pre></div>
+<span class="o">}</span></code></pre></figure>
<p>The key and value are declared as Object, and need to be cast to the correct type. If you don’t configure a <a href="../container/serialization.html">serializer/deserializer</a>, they are typically Java byte arrays. A deserializer can convert these bytes into any other type, for example the JSON deserializer mentioned above parses the byte array into java.util.Map, java.util.List and String objects.</p>
@@ -223,7 +221,7 @@
<p>The API looks like this:</p>
-<div class="highlight"><pre><code class="java"><span class="cm">/** A triple of system name, stream name and partition. */</span>
+<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="cm">/** A triple of system name, stream name and partition. */</span>
<span class="kd">public</span> <span class="kd">class</span> <span class="nc">SystemStreamPartition</span> <span class="kd">extends</span> <span class="n">SystemStream</span> <span class="o">{</span>
<span class="cm">/** The name of the system which provides this stream. It is</span>
@@ -235,16 +233,16 @@
<span class="cm">/** The partition within the stream. */</span>
<span class="kd">public</span> <span class="n">Partition</span> <span class="nf">getPartition</span><span class="o">()</span> <span class="o">{</span> <span class="o">...</span> <span class="o">}</span>
-<span class="o">}</span></code></pre></div>
+<span class="o">}</span></code></pre></figure>
<p>In the example job configuration above, the system name is “kafka”, the stream name is “PageViewEvent”. (The name “kafka” isn’t special — you can give your system any name you want.) If you have several input streams feeding into your StreamTask, you can use the SystemStreamPartition to determine what kind of message you’ve received.</p>
<p>What about sending messages? If you take a look at the process() method in StreamTask, you’ll see that you get a <a href="javadocs/org/apache/samza/task/MessageCollector.html">MessageCollector</a>.</p>
-<div class="highlight"><pre><code class="java"><span class="cm">/** When a task wishes to send a message, it uses this interface. */</span>
+<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="cm">/** When a task wishes to send a message, it uses this interface. */</span>
<span class="kd">public</span> <span class="kd">interface</span> <span class="nc">MessageCollector</span> <span class="o">{</span>
<span class="kt">void</span> <span class="nf">send</span><span class="o">(</span><span class="n">OutgoingMessageEnvelope</span> <span class="n">envelope</span><span class="o">);</span>
-<span class="o">}</span></code></pre></div>
+<span class="o">}</span></code></pre></figure>
<p>To send a message, you create an <a href="javadocs/org/apache/samza/system/OutgoingMessageEnvelope.html">OutgoingMessageEnvelope</a> object and pass it to the message collector. At a minimum, the envelope specifies the message you want to send, and the system and stream name to send it to. Optionally you can specify the partitioning key and other parameters. See the <a href="javadocs/org/apache/samza/system/OutgoingMessageEnvelope.html">javadoc</a> for details.</p>
@@ -252,12 +250,12 @@
<p>For example, here’s a simple task that splits each input message into words, and emits each word as a separate message:</p>
-<div class="highlight"><pre><code class="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SplitStringIntoWords</span> <span class="kd">implements</span> <span class="n">StreamTask</span> <span class="o">{</span>
+<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SplitStringIntoWords</span> <span class="kd">implements</span> <span class="n">StreamTask</span> <span class="o">{</span>
<span class="c1">// Send outgoing messages to a stream called "words"</span>
<span class="c1">// in the "kafka" system.</span>
<span class="kd">private</span> <span class="kd">final</span> <span class="n">SystemStream</span> <span class="n">OUTPUT_STREAM</span> <span class="o">=</span>
- <span class="k">new</span> <span class="nf">SystemStream</span><span class="o">(</span><span class="s">"kafka"</span><span class="o">,</span> <span class="s">"words"</span><span class="o">);</span>
+ <span class="k">new</span> <span class="n">SystemStream</span><span class="o">(</span><span class="s">"kafka"</span><span class="o">,</span> <span class="s">"words"</span><span class="o">);</span>
<span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span><span class="n">IncomingMessageEnvelope</span> <span class="n">envelope</span><span class="o">,</span>
<span class="n">MessageCollector</span> <span class="n">collector</span><span class="o">,</span>
@@ -270,11 +268,11 @@
<span class="n">collector</span><span class="o">.</span><span class="na">send</span><span class="o">(</span><span class="k">new</span> <span class="n">OutgoingMessageEnvelope</span><span class="o">(</span><span class="n">OUTPUT_STREAM</span><span class="o">,</span> <span class="n">word</span><span class="o">,</span> <span class="mi">1</span><span class="o">));</span>
<span class="o">}</span>
<span class="o">}</span>
-<span class="o">}</span></code></pre></div>
+<span class="o">}</span></code></pre></figure>
<p>For AsyncStreamTask example, follow the tutorial in <a href="../../../tutorials/latest/samza-async-user-guide.html">Samza Async API and Multithreading User Guide</a>. For more details on APIs, please refer to <a href="../jobs/configuration-table.html">Configuration</a> and <a href="javadocs">Javadocs</a>.</p>
-<h2 id="samzacontainer-»"><a href="../container/samza-container.html">SamzaContainer »</a></h2>
+<h2 id="samzacontainer"><a href="../container/samza-container.html">SamzaContainer »</a></h2>
</div>
@@ -292,7 +290,7 @@
<script>
$( document ).ready(function() {
- if ( $.fn.urlExists( "/learn/documentation/0.14/api/overview.html" ) ) {
+ if ( $.fn.urlExists( "/learn/documentation/0.14/api/overview" ) ) {
$("#switch-version-button").addClass("fa fa-history masthead-icon");
}
});
Modified: samza/site/learn/documentation/latest/aws/kinesis.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/aws/kinesis.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/aws/kinesis.html (original)
+++ samza/site/learn/documentation/latest/aws/kinesis.html Fri May 25 22:28:38 2018
@@ -44,9 +44,7 @@
<!-- this icon only shows in versioned pages -->
-
-
- <a href="http://samza.apache.org/learn/documentation/0.14/aws/kinesis.html"><i id="switch-version-button"></i></a>
+ <a href="http://samza.apache.org/learn/documentation/0.14/aws/kinesis"><i id="switch-version-button"></i></a>
<!-- links for the navigation bar -->
@@ -149,13 +147,13 @@
<p>You can configure your Samza jobs to process data from <a href="https://aws.amazon.com/kinesis/data-streams">AWS Kinesis</a>, Amazon’s data streaming service. A <code>Kinesis data stream</code> is similar to a Kafka topic and can have multiple partitions. Each message consumed from a Kinesis data stream is an instance of <a href="http://docs.aws.amazon.com/goto/WebAPI/kinesis-2013-12-02/Record">Record</a>.</p>
-<h3 id="consuming-from-kinesis:">Consuming from Kinesis:</h3>
+<h3 id="consuming-from-kinesis">Consuming from Kinesis:</h3>
<p>Samza’s <a href="https://github.com/apache/samza/blob/master/samza-aws/src/main/java/org/apache/samza/system/kinesis/consumer/KinesisSystemConsumer.java">KinesisSystemConsumer</a> wraps the Record into a <a href="https://github.com/apache/samza/blob/master/samza-aws/src/main/java/org/apache/samza/system/kinesis/consumer/KinesisIncomingMessageEnvelope.java">KinesisIncomingMessageEnvelope</a>. The key of the message is set to partition key of the Record. The message is obtained from the Record body.</p>
<p>To configure Samza to consume from Kinesis streams:</p>
-<div class="highlight"><pre><code class="jproperties"><span class="c"># define a kinesis system factory with your identifier. eg: kinesis-system</span>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="c"># define a kinesis system factory with your identifier. eg: kinesis-system</span>
<span class="na">systems.kinesis-system.samza.factory</span><span class="o">=</span><span class="s">org.apache.samza.system.eventhub.KinesisSystemFactory</span>
<span class="c"># kinesis system consumer works with only AllSspToSingleTaskGrouperFactory</span>
@@ -167,34 +165,34 @@
<span class="c"># define required properties for your streams</span>
<span class="na">systems.kinesis-system.streams.input0.aws.region</span><span class="o">=</span><span class="s">YOUR-STREAM-REGION</span>
<span class="na">systems.kinesis-system.streams.input0.aws.accessKey</span><span class="o">=</span><span class="s">YOUR-ACCESS_KEY</span>
-<span class="na">sensitive.systems.kinesis-system.streams.input0.aws.secretKey</span><span class="o">=</span><span class="s">YOUR-SECRET-KEY</span></code></pre></div>
+<span class="na">sensitive.systems.kinesis-system.streams.input0.aws.secretKey</span><span class="o">=</span><span class="s">YOUR-SECRET-KEY</span></code></pre></figure>
<p>The tuple required to access the Kinesis data stream must be provided, namely the fields <code>YOUR-STREAM-REGION</code>, <code>YOUR-ACCESS-KEY</code>, <code>YOUR-SECRET-KEY</code>.</p>
-<h4 id="advanced-configuration:">Advanced Configuration:</h4>
+<h4 id="advanced-configuration">Advanced Configuration:</h4>
-<h5 id="aws-client-configs:">AWS Client Configs:</h5>
+<h5 id="aws-client-configs">AWS Client Configs:</h5>
<p>You can configure any <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ClientConfiguration.html">AWS client config</a> with the prefix <code>system.system-name.aws.clientConfig.*</code></p>
-<div class="highlight"><pre><code class="jproperties"><span class="na">system.system-name.aws.clientConfig.CONFIG-NAME</span><span class="o">=</span><span class="s">CONFIG-VALUE</span></code></pre></div>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="na">system.system-name.aws.clientConfig.CONFIG-NAME</span><span class="o">=</span><span class="s">CONFIG-VALUE</span></code></pre></figure>
<p>As an example, to set a proxy host and proxy port for the AWS Client:</p>
-<div class="highlight"><pre><code class="jproperties"><span class="na">systems.system-name.aws.clientConfig.ProxyHost</span><span class="o">=</span><span class="s">my-proxy-host.com</span>
-<span class="na">systems.system-name.aws.clientConfig.ProxyPort</span><span class="o">=</span><span class="s">my-proxy-port</span></code></pre></div>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="na">systems.system-name.aws.clientConfig.ProxyHost</span><span class="o">=</span><span class="s">my-proxy-host.com</span>
+<span class="na">systems.system-name.aws.clientConfig.ProxyPort</span><span class="o">=</span><span class="s">my-proxy-port</span></code></pre></figure>
-<h5 id="kcl-configs:">KCL Configs:</h5>
+<h5 id="kcl-configs">KCL Configs:</h5>
<p>Similarly, you can set any <a href="https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/lib/worker/KinesisClientLibConfiguration.java">Kinesis Client Library config</a> for a stream by configuring it under <code>systems.system-name.streams.stream-name.aws.kcl.*</code></p>
-<div class="highlight"><pre><code class="jproperties"><span class="na">systems.system-name.streams.stream-name.aws.kcl.CONFIG-NAME</span><span class="o">=</span><span class="s">CONFIG-VALUE</span></code></pre></div>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="na">systems.system-name.streams.stream-name.aws.kcl.CONFIG-NAME</span><span class="o">=</span><span class="s">CONFIG-VALUE</span></code></pre></figure>
<p>As an example, to reset the checkpoint and set the starting position for a stream:</p>
-<div class="highlight"><pre><code class="jproperties"><span class="na">systems.kinesis-system.streams.input0.aws.kcl.TableName</span><span class="o">=</span><span class="s">my-app-table-name</span>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="na">systems.kinesis-system.streams.input0.aws.kcl.TableName</span><span class="o">=</span><span class="s">my-app-table-name</span>
<span class="c"># set the starting position to either TRIM_HORIZON (oldest) or LATEST (latest)</span>
-<span class="na">systems.kinesis-system.streams.input0.aws.kcl.InitialPositionInStream</span><span class="o">=</span><span class="s">my-start-position</span></code></pre></div>
+<span class="na">systems.kinesis-system.streams.input0.aws.kcl.InitialPositionInStream</span><span class="o">=</span><span class="s">my-start-position</span></code></pre></figure>
<h4 id="limitations">Limitations</h4>
@@ -204,11 +202,11 @@
* Kinesis streams must be used with the <a href="https://github.com/apache/samza/blob/master/samza-core/src/main/java/org/apache/samza/container/grouper/stream/AllSspToSingleTaskGrouperFactory.java">AllSspToSingleTaskGrouperFactory</a>. No other grouper is supported.
* A Samza job that consumes from Kinesis cannot consume from any other input source. However, you can send your results to any destination (eg: Kafka, EventHubs), and have another Samza job consume them.</p>
-<h3 id="producing-to-kinesis:">Producing to Kinesis:</h3>
+<h3 id="producing-to-kinesis">Producing to Kinesis:</h3>
<p>The KinesisSystemProducer for Samza is not yet implemented.</p>
-<h3 id="how-to-configure-samza-job-to-consume-from-kinesis-data-stream-?">How to configure Samza job to consume from Kinesis data stream ?</h3>
+<h3 id="how-to-configure-samza-job-to-consume-from-kinesis-data-stream">How to configure Samza job to consume from Kinesis data stream ?</h3>
<p>This tutorial uses <a href="../../../startup/hello-samza/latest/">hello samza</a> to illustrate running a Samza job on Yarn that consumes from Kinesis. We will use the <a href="https://github.com/apache/samza-hello-samza/blob/master/src/main/java/samza/examples/kinesis/KinesisHelloSamza.java">KinesisHelloSamza</a> example.</p>
@@ -216,10 +214,10 @@
<p>Update the following properties in the kinesis-hello-samza.properties file:</p>
-<div class="highlight"><pre><code class="jproperties"><span class="na">task.inputs</span><span class="o">=</span><span class="s">kinesis.<kinesis-stream></span>
+<figure class="highlight"><pre><code class="language-jproperties" data-lang="jproperties"><span></span><span class="na">task.inputs</span><span class="o">=</span><span class="s">kinesis.<kinesis-stream></span>
<span class="na">systems.kinesis.streams.<kinesis-stream>.aws.region</span><span class="o">=</span><span class="s"><kinesis-stream-region></span>
<span class="na">systems.kinesis.streams.<kinesis-stream>.aws.accessKey</span><span class="o">=</span><span class="s"><your-access-key></span>
-<span class="na">sensitive.systems.kinesis.streams.<kinesis-stream>.aws.region</span><span class="o">=</span><span class="s"><your-secret-key></span></code></pre></div>
+<span class="na">sensitive.systems.kinesis.streams.<kinesis-stream>.aws.region</span><span class="o">=</span><span class="s"><your-secret-key></span></code></pre></figure>
<p>Now, you are ready to run your Samza application on Yarn as described <a href="../../../startup/hello-samza/latest/">here</a>. Check the log file for messages read from your Kinesis stream.</p>
@@ -239,7 +237,7 @@
<script>
$( document ).ready(function() {
- if ( $.fn.urlExists( "/learn/documentation/0.14/aws/kinesis.html" ) ) {
+ if ( $.fn.urlExists( "/learn/documentation/0.14/aws/kinesis" ) ) {
$("#switch-version-button").addClass("fa fa-history masthead-icon");
}
});
Modified: samza/site/learn/documentation/latest/azure/eventhubs.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/azure/eventhubs.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/azure/eventhubs.html (original)
+++ samza/site/learn/documentation/latest/azure/eventhubs.html Fri May 25 22:28:38 2018
@@ -44,9 +44,7 @@
<!-- this icon only shows in versioned pages -->
-
-
- <a href="http://samza.apache.org/learn/documentation/0.14/azure/eventhubs.html"><i id="switch-version-button"></i></a>
+ <a href="http://samza.apache.org/learn/documentation/0.14/azure/eventhubs"><i id="switch-version-button"></i></a>
<!-- links for the navigation bar -->
@@ -149,12 +147,12 @@
<p>You can configure your Samza jobs to process data from <a href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-features">Azure Eventhubs</a>, Microsoft’s data streaming service. An <code>event hub</code> is similar to a Kafka topic and can have multiple partitions with producers and consumers. Each message produced or consumed from an event hub is an instance of <a href="https://docs.microsoft.com/en-us/java/api/com.microsoft.azure.eventhubs._event_data">EventData</a>. </p>
-<h3 id="consuming-from-eventhubs:">Consuming from EventHubs:</h3>
+<h3 id="consuming-from-eventhubs">Consuming from EventHubs:</h3>
<p>Samza’s <a href="https://github.com/apache/samza/blob/master/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubSystemConsumer.java">EventHubSystemConsumer</a> wraps the EventData into an <a href="https://github.com/apache/samza/blob/master/samza-azure/src/main/java/org/apache/samza/system/eventhub/consumer/EventHubIncomingMessageEnvelope.java">EventHubIncomingMessageEnvelope</a>. The key of the message is set to the partition key of the EventData. The message is obtained from the EventData body. </p>
<p>To configure Samza to configure from EventHub streams: </p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"># define an event hub system factory with your identifier. eg: eh-system
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span># define an event hub system factory with your identifier. eg: eh-system
systems.eh-system.samza.factory=org.apache.samza.system.eventhub.EventHubSystemFactory
# define your streams
@@ -173,7 +171,7 @@ systems.eh-system.streams.output0.eventh
</code></pre></div>
<p>The tuple required to access the Eventhubs entity per stream must be provided, namely the fields <code>YOUR-STREAM-NAMESPACE</code>, <code>YOUR-ENTITY-NAME</code>, <code>YOUR-SAS-KEY-NAME</code>, <code>YOUR-SAS-KEY-TOKEN</code>.</p>
-<h3 id="producing-to-eventhubs:">Producing to EventHubs:</h3>
+<h3 id="producing-to-eventhubs">Producing to EventHubs:</h3>
<p>Similarly, you can also configure your Samza job to write to EventHubs.<br>
<code>
@@ -183,9 +181,13 @@ collector.send(envelope);
<p>Each <a href="https://samza.apache.org/learn/documentation/latest/api/javadocs/org/apache/samza/system/OutgoingMessageEnvelope.html">OutgoingMessageEnvelope</a> is converted into an <a href="https://docs.microsoft.com/en-us/java/api/com.microsoft.azure.eventhubs._event_data">EventData</a> instance whose body is set to the <code>message</code> in the envelope. Additionally, the <code>key</code> and the <code>produce timestamp</code> are set as properties in the EventData before sending it to EventHubs.</p>
-<h3 id="advanced-configuration:">Advanced configuration:</h3>
+<h4 id="size-limit-of-partition-key">Size limit of partition key:</h4>
+
+<p>Note that EventHubs has a limit on the length of partition key (128 characters). In <a href="https://github.com/apache/samza/blob/master/samza-azure/src/main/java/org/apache/samza/system/eventhub/producer/EventHubSystemProducer.java">EventHubSystemProducer</a> we truncate the partition key if the size of the key exceeds the limit.</p>
+
+<h3 id="advanced-configuration">Advanced configuration:</h3>
-<h5 id="producer-partitioning:">Producer partitioning:</h5>
+<h5 id="producer-partitioning">Producer partitioning:</h5>
<p>The <code>partition.method</code> property determines how outgoing messages are partitioned. Valid values for this config are <code>EVENT_HUB_HASHING</code>, <code>PARTITION_KEY_AS_PARTITION</code> or <code>ROUND_ROBIN</code>. </p>
@@ -196,23 +198,23 @@ collector.send(envelope);
<p><code>ROUND_ROBIN</code>: In this method, outgoing messages are distributed in a round-robin across all partitions. The key and the partition key in the message are ignored.</p>
<p><img src="/img/latest/learn/documentation/azure/eventhub_send_methods.png" alt="diagram-medium"></p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">systems.eh-system.partition.method = EVENT_HUB_HASHING
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>systems.eh-system.partition.method = EVENT_HUB_HASHING
</code></pre></div>
-<h5 id="consumer-groups:">Consumer groups:</h5>
+<h5 id="consumer-groups">Consumer groups:</h5>
<p>Eventhub supports a notion of <a href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-features#consumer-groups">consumer groups</a> which enable multiple applications have their own view of the event stream. Each event hub stream has a pre-defined consumer group named <code>$Default</code>. You can define your own consumer group for your job and configure a <code>eventhubs.consumer.group</code> </p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">systems.eh-system.streams.eh-input0.eventhubs.consumer.group = my-group
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>systems.eh-system.streams.eh-input0.eventhubs.consumer.group = my-group
</code></pre></div>
-<h5 id="serde:">Serde:</h5>
+<h5 id="serde">Serde:</h5>
<p>By default, the messages from EventHubs are sent and received as byte arrays. You can configure a serializer and deserializer for your message by setting a value for <code>msg.serde</code> for your stream. </p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">streams.input0.samza.msg.serde = json
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>streams.input0.samza.msg.serde = json
streams.output0.samza.msg.serde = json
</code></pre></div>
-<h5 id="consumer-buffer-size:">Consumer buffer size:</h5>
+<h5 id="consumer-buffer-size">Consumer buffer size:</h5>
<p>When the consumer reads a message from event hubs, it appends them to a shared producer-consumer buffer corresponding to its partition. This config determines the per-partition queue size. Setting a higher value for this config typically achieves a higher throughput at the expense of increased on-heap memory.</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">systems.eh-system.eventhubs.receive.queue.size = 10
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>systems.eh-system.eventhubs.receive.queue.size = 10
</code></pre></div>
<p>For the list of all configs, check out the configuration table page <a href="../jobs/configuration-table.html">here</a></p>
@@ -223,7 +225,7 @@ streams.output0.samza.msg.serde = json
<h4 id="get-the-code">Get the Code</h4>
<p>Let’s get started by cloning the hello-samza project</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">git clone https://git.apache.org/samza-hello-samza.git hello-samza
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>git clone https://git.apache.org/samza-hello-samza.git hello-samza
cd hello-samza
git checkout latest
</code></pre></div>
@@ -232,7 +234,7 @@ git checkout latest
<h4 id="setting-up-the-deployment-environment">Setting up the Deployment Environment</h4>
<p>For our Azure application, we require <a href="http://zookeeper.apache.org/">ZooKeeper</a>. The hello-samza project comes with a script called “grid” to help with the environment setup</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">./bin/grid standalone
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>./bin/grid standalone
</code></pre></div>
<p>This command will download, install, and start ZooKeeper and Kafka. It will also check out the latest version of Samza and build it. All package files will be put in a sub-directory called “deploy” inside hello-samza’s root folder.</p>
@@ -241,7 +243,7 @@ git checkout latest
<h4 id="configuring-the-azure-application">Configuring the Azure application</h4>
<p>Here are the configs you must set before building the project. Configure these in the <code>src/main/config/azure-application-local-runner.properties</code> file.</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"># Add your EventHubs input stream credentials here
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span># Add your EventHubs input stream credentials here
systems.eventhubs.streams.input-stream.eventhubs.namespace=YOUR-STREAM-NAMESPACE
systems.eventhubs.streams.input-stream.eventhubs.entitypath=YOUR-ENTITY-NAME
systems.eventhubs.streams.input-stream.eventhubs.sas.keyname=YOUR-SAS-KEY-NAME
@@ -254,23 +256,23 @@ systems.eventhubs.streams.output-stream.
systems.eventhubs.streams.output-stream.eventhubs.sas.token=YOUR-SAS-KEY-TOKEN
</code></pre></div>
<p>Optionally, you may also use the Azure Checkpoint Manager. Otherwise, comment out both these lines.</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text"># Azure Table Checkpoint Manager
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span># Azure Table Checkpoint Manager
task.checkpoint.factory=org.apache.samza.checkpoint.azure.AzureCheckpointManagerFactory
azure.storage.connect=YOUR-STORAGE-ACCOUNT-CONNECTION-STRING
</code></pre></div>
<h4 id="building-the-hello-samza-project">Building the Hello Samza Project</h4>
<p>With the environment setup complete, let us move on to building the hello-samza project. Execute the following commands:</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">mvn clean package
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>mvn clean package
mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.14.1-SNAPSHOT-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.14.1-dist.tar.gz -C deploy/samza
</code></pre></div>
<p>We are now all set to deploy the application locally.</p>
<h4 id="running-the-azure-application">Running the Azure application</h4>
<p>In order to run the application, we will use the <em>run-azure-application</em> script.</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">./deploy/samza/bin/run-azure-application.sh
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>./deploy/samza/bin/run-azure-application.sh
</code></pre></div>
<p>The above command executes the helper script which invokes the <em>AzureZKLocalApplication</em> main class, which starts the <em>AzureApplication</em>. This application filters out the messages consumed without keys, prints them out and send them the configured output stream.</p>
@@ -285,7 +287,7 @@ Received Message: <VALUE>
<p>This application can be shutdown by terminating the <em>run-azure-application</em> script.
We can use the <em>grid</em> script to tear down the local environment (<a href="http://kafka.apache.org/">Kafka</a> and <a href="http://zookeeper.apache.org/">Zookeeper</a>).</p>
-<div class="highlight"><pre><code class="language-text" data-lang="text">bin/grid stop all
+<div class="highlight"><pre><code class="language-text" data-lang="text"><span></span>bin/grid stop all
</code></pre></div>
</div>
@@ -303,7 +305,7 @@ We can use the <em>grid</em> script to t
<script>
$( document ).ready(function() {
- if ( $.fn.urlExists( "/learn/documentation/0.14/azure/eventhubs.html" ) ) {
+ if ( $.fn.urlExists( "/learn/documentation/0.14/azure/eventhubs" ) ) {
$("#switch-version-button").addClass("fa fa-history masthead-icon");
}
});
Modified: samza/site/learn/documentation/latest/comparisons/introduction.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/comparisons/introduction.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/comparisons/introduction.html (original)
+++ samza/site/learn/documentation/latest/comparisons/introduction.html Fri May 25 22:28:38 2018
@@ -44,9 +44,7 @@
<!-- this icon only shows in versioned pages -->
-
-
- <a href="http://samza.apache.org/learn/documentation/0.14/comparisons/introduction.html"><i id="switch-version-button"></i></a>
+ <a href="http://samza.apache.org/learn/documentation/0.14/comparisons/introduction"><i id="switch-version-button"></i></a>
<!-- links for the navigation bar -->
@@ -167,7 +165,7 @@
<p>MapReduce is sometimes criticized for writing to disk more than necessary. However, this criticism applies less to stream processing: batch processing like MapReduce often is used for processing large historical collections of data in a short period of time (e.g. query a month of data in ten minutes), whereas stream processing mostly needs to keep up with the steady-state flow of data (process 10 minutes worth of data in 10 minutes). This means that the raw throughput requirements for stream processing are, generally, orders of magnitude lower than for batch processing.</p>
-<h3 id="-state"><a name="state"></a> State</h3>
+<h3 id="state"><a name="state"></a> State</h3>
<p>Only the very simplest stream processing problems are stateless (i.e. can process one message at a time, independently of all other messages). Many stream processing applications require a job to maintain some state. For example:</p>
@@ -207,7 +205,7 @@ example above, where you have a stream o
<p>We think there will be a lot of innovation both in open source virtualization frameworks like Mesos and YARN and in commercial cloud providers like Amazon, so it makes sense to integrate with them.</p>
-<h2 id="mupd8-»"><a href="mupd8.html">MUPD8 »</a></h2>
+<h2 id="mupd8"><a href="mupd8.html">MUPD8 »</a></h2>
</div>
@@ -225,7 +223,7 @@ example above, where you have a stream o
<script>
$( document ).ready(function() {
- if ( $.fn.urlExists( "/learn/documentation/0.14/comparisons/introduction.html" ) ) {
+ if ( $.fn.urlExists( "/learn/documentation/0.14/comparisons/introduction" ) ) {
$("#switch-version-button").addClass("fa fa-history masthead-icon");
}
});
Modified: samza/site/learn/documentation/latest/comparisons/mupd8.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/comparisons/mupd8.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/comparisons/mupd8.html (original)
+++ samza/site/learn/documentation/latest/comparisons/mupd8.html Fri May 25 22:28:38 2018
@@ -44,9 +44,7 @@
<!-- this icon only shows in versioned pages -->
-
-
- <a href="http://samza.apache.org/learn/documentation/0.14/comparisons/mupd8.html"><i id="switch-version-button"></i></a>
+ <a href="http://samza.apache.org/learn/documentation/0.14/comparisons/mupd8"><i id="switch-version-button"></i></a>
<!-- links for the navigation bar -->
@@ -165,7 +163,7 @@
<p>MUPD8 buffers messages in an in-memory queue when passing messages between two MUPD8 tasks. When a queue fills up, developers have the option to either drop the messages on the floor, log the messages to local disk, or block until the queue frees up. All of these options are sub-optimal. Dropping messages leads to incorrect results. Blocking your stream processor can have a cascading effect, where the slowest processor blocks all upstream processors, which in turn block their upstream processors, until the whole system grinds to a halt. Logging to local disk is the most reasonable, but when a fault occurs, those messages are lost on failover.</p>
-<p>By adopting Kafka’s broker as a remote buffer, Samza solves all of these problems. It doesn’t need to block because consumers and producers are decoupled using the Kafka brokers' disks as buffers. Messages are not dropped because Kafka brokers are highly available as of version 0.8. In the event of a failure, when a Samza job is restarted on another machine, its input and output are not lost, because they are stored remotely on replicated Kafka brokers.</p>
+<p>By adopting Kafka’s broker as a remote buffer, Samza solves all of these problems. It doesn’t need to block because consumers and producers are decoupled using the Kafka brokers’ disks as buffers. Messages are not dropped because Kafka brokers are highly available as of version 0.8. In the event of a failure, when a Samza job is restarted on another machine, its input and output are not lost, because they are stored remotely on replicated Kafka brokers.</p>
<h3 id="state-management">State Management</h3>
@@ -213,7 +211,7 @@
<p>The MUPD8 team has published a very good <a href="http://vldb.org/pvldb/vol5/p1814_wanglam_vldb2012.pdf">paper</a> on the design of their system.</p>
-<h2 id="storm-»"><a href="storm.html">Storm »</a></h2>
+<h2 id="storm"><a href="storm.html">Storm »</a></h2>
</div>
@@ -231,7 +229,7 @@
<script>
$( document ).ready(function() {
- if ( $.fn.urlExists( "/learn/documentation/0.14/comparisons/mupd8.html" ) ) {
+ if ( $.fn.urlExists( "/learn/documentation/0.14/comparisons/mupd8" ) ) {
$("#switch-version-button").addClass("fa fa-history masthead-icon");
}
});
Modified: samza/site/learn/documentation/latest/comparisons/spark-streaming.html
URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/latest/comparisons/spark-streaming.html?rev=1832281&r1=1832280&r2=1832281&view=diff
==============================================================================
--- samza/site/learn/documentation/latest/comparisons/spark-streaming.html (original)
+++ samza/site/learn/documentation/latest/comparisons/spark-streaming.html Fri May 25 22:28:38 2018
@@ -44,9 +44,7 @@
<!-- this icon only shows in versioned pages -->
-
-
- <a href="http://samza.apache.org/learn/documentation/0.14/comparisons/spark-streaming.html"><i id="switch-version-button"></i></a>
+ <a href="http://samza.apache.org/learn/documentation/0.14/comparisons/spark-streaming"><i id="switch-version-button"></i></a>
<!-- links for the navigation bar -->
@@ -204,7 +202,7 @@
<p>In Samza, you have the flexibility to define what one task contains. For example, in the Kafka use case, by default, Samza groups the partitions with the same partition id into one task. This allows easy join between different streams. Out-of-box, Samza also provides the grouping strategy which assigns one partition to one task. This provides maximum scalability in terms of how many containers can be used to process those input streams and is appropriate for very high volume jobs that need no grouping of the input streams.</p>
-<h3 id="buffering-&-latency">Buffering & Latency</h3>
+<h3 id="buffering-latency">Buffering & Latency</h3>
<p>Spark streaming essentially is a sequence of small batch processes. With a fast execution engine, it can reach the latency as low as one second (from their <a href="http://www.cs.berkeley.edu/%7Ematei/papers/2012/hotcloud_spark_streaming.pdf">paper</a>). From their <a href="https://spark.apache.org/docs/latest/streaming-programming-guide.html#level-of-parallelism-in-data-receiving">page</a>, “the recommended minimum value of block interval is about 50 ms, below which the task launching overheads may be a problem.”</p>
@@ -220,7 +218,7 @@
<p>In Samza, YARN takes care of the fault-tolerance. When the AM fails in Samza, YARN will handle restarting the AM. Samza will restart all the containers if the AM restarts. When the container fails, the AM will bring up a new container.</p>
-<h3 id="deployment-&-execution">Deployment & Execution</h3>
+<h3 id="deployment-execution">Deployment & Execution</h3>
<p>Spark has a SparkContext object to talk with cluster managers, which then allocate resources for the application. Currently Spark supports three types of cluster managers: <a href="http://spark.apache.org/docs/latest/spark-standalone.html">Spark standalone</a>, <a href="http://mesos.apache.org/">Apache Mesos</a> and <a href="http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html">Hadoop YARN</a>. Besides these, Spark has a script for launching in <a href="http://spark.apache.org/docs/latest/ec2-scripts.html">Amazon EC2</a>.</p>
@@ -248,7 +246,7 @@
<p>Samza is still young, but has just released version 0.9.0. It has a responsive community and is being developed actively. That said, it is built on solid systems such as YARN and Kafka. Samza is heavily used at LinkedIn and <a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By">other companies</a>. we hope others will find it useful as well.</p>
-<h2 id="api-overview-»"><a href="../api/overview.html">API Overview »</a></h2>
+<h2 id="api-overview"><a href="../api/overview.html">API Overview »</a></h2>
</div>
@@ -266,7 +264,7 @@
<script>
$( document ).ready(function() {
- if ( $.fn.urlExists( "/learn/documentation/0.14/comparisons/spark-streaming.html" ) ) {
+ if ( $.fn.urlExists( "/learn/documentation/0.14/comparisons/spark-streaming" ) ) {
$("#switch-version-button").addClass("fa fa-history masthead-icon");
}
});