You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@distributedlog.apache.org by si...@apache.org on 2017/04/26 18:56:55 UTC
[32/51] [partial] incubator-distributedlog git commit: Release
0.4.0-incubating
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ef7245e8/content/docs/0.4.0-incubating/user_guide/configuration/perlog.html
----------------------------------------------------------------------
diff --git a/content/docs/0.4.0-incubating/user_guide/configuration/perlog.html b/content/docs/0.4.0-incubating/user_guide/configuration/perlog.html
new file mode 100644
index 0000000..aeabc29
--- /dev/null
+++ b/content/docs/0.4.0-incubating/user_guide/configuration/perlog.html
@@ -0,0 +1,747 @@
+<!DOCTYPE html>
+<html lang="en">
+
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Apache DistributedLog (incubating)</title>
+ <meta name="description" content="Apache DistributedLog is an high performance replicated log.
+">
+
+ <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css">
+ <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css">
+ <!-- JQuery -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
+ <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script>
+ <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/user_guide/configuration/perlog.html" data-proofer-ignore>
+ <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml">
+ <!-- Font Awesome -->
+ <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script>
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-83870961-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+ <!-- End Google Analytics -->
+ <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
+</head>
+
+
+ <body role="document">
+
+
+<nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <a href="/" class="navbar-brand" >
+ <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png">
+ </a>
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ </div>
+ <div id="navbar" class="navbar-collapse collapse">
+ <ul class="nav navbar-nav">
+ <!-- Overview -->
+ <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li>
+ <!-- Concepts -->
+ <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li>
+ <!-- Quick Start -->
+ <li>
+ <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/building.html">
+ Build DistributedLog from Source
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/download.html">
+ Download Releases
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Quickstart</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/quickstart.html">
+ Setup & Run Example
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-1.html">
+ API - Write Records (via core library)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-2.html">
+ API - Write Records (via write proxy)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-5.html">
+ API - Read Records
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Deployment</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/cluster.html">
+ Cluster Setup
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/global-cluster.html">
+ Global Cluster Setup
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/docker.html">
+ Docker
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- API -->
+ <li>
+ <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li>
+ </ul>
+ </li>
+ <!-- User Guide -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/basics/introduction.html">
+ Introduction
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html">
+ Considerations
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html">
+ Architecture
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/main.html">
+ API
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html">
+ Configuration
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/design/main.html">
+ Detail Design
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html">
+ Global Replicated Log
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html">
+ Implementation
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/main.html">
+ References
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- Admin Guide -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/operations.html">
+ Operations
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/performance.html">
+ Performance Tuning
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html">
+ Load Test
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/hardware.html">
+ Hardware
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html">
+ Monitoring
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html">
+ ZooKeeper
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html">
+ BookKeeper
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- Tutorials -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li class="dropdown-header"><strong>Basic</strong></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Messaging</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html">
+ Write records to partitioned streams
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html">
+ Write records to multiple streams (load balancer)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html">
+ At-least-once Processing
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html">
+ Exact-Once Processing
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html">
+ Implement a kafka-like pub/sub system
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Replicated State Machines</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html">
+ Build replicated state machines
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Analytics</strong></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li>
+ </ul>
+ </li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+</nav>
+
+
+<link rel="stylesheet" href="">
+
+
+ <div class="container" role="main">
+
+ <div class="row">
+
+ <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<div class="row">
+ <!-- Sub Navigation -->
+ <div class="col-sm-3">
+ <ul id="sub-nav">
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/main.html" class="">User Guide</a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/basics/introduction.html" class="">
+ Introduction
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html" class="">
+ Considerations
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html" class="">
+ Architecture
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/main.html" class="">
+ API
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/core.html" class="active">
+ Core Library API
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/proxy.html" class="active">
+ Proxy Client API
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/practice.html" class="active">
+ Best Practise
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html" class="">
+ Configuration
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/core.html" class="active">
+ Core Library Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/proxy.html" class="active">
+ Write Proxy Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/client.html" class="active">
+ Client Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/perlog.html" class="active">
+ Per Stream Configuration
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/design/main.html" class="">
+ Detail Design
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html" class="">
+ Global Replicated Log
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html" class="">
+ Implementation
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/storage.html" class="active">
+ Storage
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/main.html" class="">
+ References
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/metrics.html" class="active">
+ Metrics
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/features.html" class="active">
+ Available Features
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+ </ul>
+
+ </li>
+
+ </ul>
+ </div>
+ <!-- Main -->
+ <div class="col-sm-9">
+ <!-- Top anchor -->
+ <a href="#top"></a>
+
+ <!-- Breadcrumbs above the main heading -->
+ <ol class="breadcrumb">
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/main.html">User Guide</a></li>
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/configuration/main.html">Configuration</a></li>
+
+
+ <li class="active">Per Stream Configuration</li>
+ </ol>
+
+ <div class="text">
+ <!-- Content -->
+ <div class="contents topic" id="per-stream-configuration">
+<p class="topic-title first">Per Stream Configuration</p>
+<ul class="simple">
+<li><a class="reference internal" href="#id1" id="id2">Per Stream Configuration</a><ul>
+<li><a class="reference internal" href="#dynamic-configuration" id="id3">Dynamic Configuration</a><ul>
+<li><a class="reference internal" href="#file-based-dynamic-configuration" id="id4">File Based Dynamic Configuration</a></li>
+<li><a class="reference internal" href="#stream-config-provider" id="id5">Stream Config Provider</a></li>
+<li><a class="reference internal" href="#available-dynamic-settings" id="id6">Available Dynamic Settings</a></li>
+<li><a class="reference internal" href="#storage-settings" id="id7">Storage Settings</a></li>
+<li><a class="reference internal" href="#transmit-settings" id="id8">Transmit Settings</a></li>
+<li><a class="reference internal" href="#durability-settings" id="id9">Durability Settings</a></li>
+<li><a class="reference internal" href="#readahead-settings" id="id10">ReadAhead Settings</a></li>
+<li><a class="reference internal" href="#rate-limit-settings" id="id11">Rate Limit Settings</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+<div class="section" id="id1">
+<h2><a class="toc-backref" href="#id2">Per Stream Configuration</a></h2>
+<p>Application is allowed to override <cite>DistributedLogConfiguration</cite> for individual streams. This is archieved
+for supplying an overrided <cite>DistributedLogConfiguration</cite> when opening the distributedlog manager.</p>
+<pre class="literal-block">
+DistributedLogNamespace namespace = ...;
+DistributedLogConfiguration perStreamConf = new DistributeLogConfiguration();
+perStreamConf.loadConf(...); // load configuration from a per stream configuration file
+DistributedLogManager dlm = namespace.openLog("test-stream", Optional.of(perStreamConf), Optional.absent());
+</pre>
+<div class="section" id="dynamic-configuration">
+<h3><a class="toc-backref" href="#id3">Dynamic Configuration</a></h3>
+<p>Besides overriding normal <cite>DistributedLogConfiguration</cite> with per stream configuration, DistributedLog also
+provides loading some configuration settings dynamically. The per stream dynamic settings are offered in
+<cite>DynamicDistributedLogConfiguration</cite>.</p>
+<div class="section" id="file-based-dynamic-configuration">
+<h4><a class="toc-backref" href="#id4">File Based Dynamic Configuration</a></h4>
+<p>The default dynamic configuration implementation is based on properties files and reloading the file periodically.</p>
+<pre class="literal-block">
+ConcurrentBaseConfiguration defaultConf = ...; // base config to fall through
+int reloadPeriod = 60; // 60 seconds
+TimeUnit reloadUnit = TimeUnit.SECOND;
+String configPath = "/path/to/per/stream/config/file";
+File configFile = new File(configPath);
+// load the fie into a properties configuration builder
+PropertiesConfigurationBuilder properties =
+ new PropertiesConfigurationBuilder(configFile.toURI().toURL());
+// Construct the dynamic configuration
+DynamicDistributedLogConfiguration dynConf = new DynamicDistributedLogConfiguration(defaultConf);
+// add a configuration subscription to periodically reload the config from the file
+ConfigurationSubscription subscription =
+ new ConfigurationSubscription(dynConf, properties, executorService, reloadPeriod, reloadUnit);
+</pre>
+</div>
+<div class="section" id="stream-config-provider">
+<h4><a class="toc-backref" href="#id5">Stream Config Provider</a></h4>
+<p>The stream config provider is designed to manage and reload dynamic configs for individual streams.</p>
+<pre class="literal-block">
+String perStreamConfigDir = "/path/to/per/stream/config/dir";
+String defaultConfigPath = "/path/to/default/config/file";
+StreamPartitionConverter converter = ...;
+ScheduledExecutorService scheduler = ...;
+int reloadPeriod = 60; // 60 seconds
+TimeUnit reloadUnit = TimeUnit.SECOND;
+StreamConfigProvider provider = new ServiceStreamConfigProvider(
+ perStreamConfigDir,
+ defaultConfigPath,
+ converter,
+ scheduler,
+ reloadPeriod,
+ reloadUnit);
+
+Optional<DynamicDistributedLogConfiguration> streamConf = provider.getDynamicStreamConfig("test-stream");
+</pre>
+<ul class="simple">
+<li><em>perStreamConfigDir</em>: The directory contains configuration files for each stream. the file name is <cite><stream_name>.conf</cite>.</li>
+<li><em>defaultConfigPath</em>: The default configuration file. If there is no stream configuration file found in <cite>perStreamConfigDir</cite>,
+it would load the configuration from <cite>defaultConfigPath</cite>.</li>
+<li><em>StreamPartitionConverter</em>: A converter that convert the stream names to partitions. DistributedLog doesn't provide built-in
+partitions. It leaves partition strategy to application. Application usually put the partition id in the dl stream name. So the
+converter is for group the streams and apply same configuration. For example, if application uses 3 streams and names them as
+<cite>test-stream_000001</cite>, <cite>test-stream_000002</cite> and <cite>test-stream_000003</cite>, a <cite>StreamPartitionConverter</cite> could be used to categorize them
+as partitions for stream <cite>test-stream</cite> and apply the configuration from file <cite>test-stream.conf</cite>.</li>
+<li><em>scheduler</em>: The executor service that reloads configuration periodically.</li>
+<li><em>reloadPeriod</em>: The reload period, in <cite>reloadUnit</cite>.</li>
+</ul>
+</div>
+<div class="section" id="available-dynamic-settings">
+<h4><a class="toc-backref" href="#id6">Available Dynamic Settings</a></h4>
+</div>
+<div class="section" id="storage-settings">
+<h4><a class="toc-backref" href="#id7">Storage Settings</a></h4>
+<ul class="simple">
+<li><em>logSegmentRetentionHours</em>: The log segment retention period, in hours. In other words, how long should DL keep the log segment once it is <cite>truncated</cite> or <cite>completed</cite>.</li>
+<li><em>bkcEnsembleSize</em>: The ensemble size of the log segment. The default value is 3.</li>
+<li><em>bkcWriteQuorumSize</em>: The write quorum size of the log segment. The default value is 3.</li>
+<li><em>bkcAckQuorumSize</em>: The ack quorumm size of the log segment. The default value is 2.</li>
+</ul>
+</div>
+<div class="section" id="transmit-settings">
+<h4><a class="toc-backref" href="#id8">Transmit Settings</a></h4>
+<ul class="simple">
+<li><em>writerOutputBufferSize</em>: The output buffer size in bytes. Larger buffer size will result in higher compression ratio and
+it would reduce the entries sent to bookkeeper, use the disk bandwidth more efficiently and improve throughput.
+Set this setting to <cite>0</cite> will ask DL to transmit the data immediately, which it would achieve low latency.</li>
+</ul>
+</div>
+<div class="section" id="durability-settings">
+<h4><a class="toc-backref" href="#id9">Durability Settings</a></h4>
+<ul class="simple">
+<li><em>isDurableWriteEnabled</em>: The flag indicates whether durable write is enabled. By default it is true.</li>
+</ul>
+</div>
+<div class="section" id="readahead-settings">
+<h4><a class="toc-backref" href="#id10">ReadAhead Settings</a></h4>
+<ul class="simple">
+<li><em>readAheadMaxRecords</em>: The maximum number of records that will be cached in readahead cache by the DL readers. The default value
+is 10. A higher value will improve throughput but use more memory. It should be tuned properly to avoid jvm gc if the reader cannot
+keep up with the writing rate.</li>
+<li><em>readAheadBatchSize</em>: The maximum number of entries that readahead worker will read in one batch. The default value is 2.
+Increase the value to increase the concurrency of reading entries from bookkeeper. It is recommended to tune to a proper value for
+catching up readers, not to exhaust bookkeeper's bandwidth.</li>
+</ul>
+</div>
+<div class="section" id="rate-limit-settings">
+<h4><a class="toc-backref" href="#id11">Rate Limit Settings</a></h4>
+<p>All the rate limit settings have both <cite>soft</cite> and <cite>hard</cite> thresholds. If the throughput goes above <cite>soft</cite> limit,
+the requests won't be rejected but just logging in the stat. But if the throughput goes above <cite>hard</cite> limit,
+the requests would be rejected immediately.</p>
+<p>NOTE: <cite>bps</cite> stands for <cite>bytes per second</cite>, while <cite>rps</cite> stands for <cite>requests per second</cite>.</p>
+<ul class="simple">
+<li><em>bpsSoftWriteLimit</em>: The soft limit for bps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+<li><em>bpsHardWriteLimit</em>: The hard limit for bps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+<li><em>rpsSoftWriteLimit</em>: The soft limit for rps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+<li><em>rpsHardWriteLimit</em>: The hard limit for rps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+</ul>
+</div>
+</div>
+</div>
+
+
+ </div>
+ </div>
+</div>
+
+
+
+ </div>
+
+
+ <hr>
+ <div class="row">
+ <div class="col-xs-12">
+ <footer>
+ <p class="text-center">© Copyright 2016
+ <a href="http://www.apache.org">The Apache Software Foundation.</a> All Rights Reserved.
+ </p>
+ <p class="text-center">
+ <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a>
+ </p>
+ </footer>
+ </div>
+ </div>
+ <!-- container div end -->
+</div>
+
+
+ <script>
+ (function () {
+ 'use strict';
+ anchors.options.placement = 'right';
+ anchors.add();
+ })();
+</script>
+
+ </body>
+
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ef7245e8/content/docs/0.4.0-incubating/user_guide/configuration/proxy.html
----------------------------------------------------------------------
diff --git a/content/docs/0.4.0-incubating/user_guide/configuration/proxy.html b/content/docs/0.4.0-incubating/user_guide/configuration/proxy.html
new file mode 100644
index 0000000..4cfefce
--- /dev/null
+++ b/content/docs/0.4.0-incubating/user_guide/configuration/proxy.html
@@ -0,0 +1,685 @@
+<!DOCTYPE html>
+<html lang="en">
+
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Apache DistributedLog (incubating)</title>
+ <meta name="description" content="Apache DistributedLog is an high performance replicated log.
+">
+
+ <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css">
+ <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css">
+ <!-- JQuery -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
+ <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script>
+ <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/user_guide/configuration/proxy.html" data-proofer-ignore>
+ <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml">
+ <!-- Font Awesome -->
+ <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script>
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-83870961-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+ <!-- End Google Analytics -->
+ <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
+</head>
+
+
+ <body role="document">
+
+
+<nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <a href="/" class="navbar-brand" >
+ <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png">
+ </a>
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ </div>
+ <div id="navbar" class="navbar-collapse collapse">
+ <ul class="nav navbar-nav">
+ <!-- Overview -->
+ <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li>
+ <!-- Concepts -->
+ <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li>
+ <!-- Quick Start -->
+ <li>
+ <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/building.html">
+ Build DistributedLog from Source
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/download.html">
+ Download Releases
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Quickstart</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/quickstart.html">
+ Setup & Run Example
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-1.html">
+ API - Write Records (via core library)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-2.html">
+ API - Write Records (via write proxy)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-5.html">
+ API - Read Records
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Deployment</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/cluster.html">
+ Cluster Setup
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/global-cluster.html">
+ Global Cluster Setup
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/docker.html">
+ Docker
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- API -->
+ <li>
+ <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li>
+ </ul>
+ </li>
+ <!-- User Guide -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/basics/introduction.html">
+ Introduction
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html">
+ Considerations
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html">
+ Architecture
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/main.html">
+ API
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html">
+ Configuration
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/design/main.html">
+ Detail Design
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html">
+ Global Replicated Log
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html">
+ Implementation
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/main.html">
+ References
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- Admin Guide -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/operations.html">
+ Operations
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/performance.html">
+ Performance Tuning
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html">
+ Load Test
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/hardware.html">
+ Hardware
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html">
+ Monitoring
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html">
+ ZooKeeper
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html">
+ BookKeeper
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- Tutorials -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li class="dropdown-header"><strong>Basic</strong></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Messaging</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html">
+ Write records to partitioned streams
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html">
+ Write records to multiple streams (load balancer)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html">
+ At-least-once Processing
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html">
+ Exact-Once Processing
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html">
+ Implement a kafka-like pub/sub system
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Replicated State Machines</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html">
+ Build replicated state machines
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Analytics</strong></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li>
+ </ul>
+ </li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+</nav>
+
+
+<link rel="stylesheet" href="">
+
+
+ <div class="container" role="main">
+
+ <div class="row">
+
+ <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<div class="row">
+ <!-- Sub Navigation -->
+ <div class="col-sm-3">
+ <ul id="sub-nav">
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/main.html" class="">User Guide</a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/basics/introduction.html" class="">
+ Introduction
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html" class="">
+ Considerations
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html" class="">
+ Architecture
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/main.html" class="">
+ API
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/core.html" class="active">
+ Core Library API
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/proxy.html" class="active">
+ Proxy Client API
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/practice.html" class="active">
+ Best Practise
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html" class="">
+ Configuration
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/core.html" class="active">
+ Core Library Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/proxy.html" class="active">
+ Write Proxy Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/client.html" class="active">
+ Client Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/perlog.html" class="active">
+ Per Stream Configuration
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/design/main.html" class="">
+ Detail Design
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html" class="">
+ Global Replicated Log
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html" class="">
+ Implementation
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/storage.html" class="active">
+ Storage
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/main.html" class="">
+ References
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/metrics.html" class="active">
+ Metrics
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/features.html" class="active">
+ Available Features
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+ </ul>
+
+ </li>
+
+ </ul>
+ </div>
+ <!-- Main -->
+ <div class="col-sm-9">
+ <!-- Top anchor -->
+ <a href="#top"></a>
+
+ <!-- Breadcrumbs above the main heading -->
+ <ol class="breadcrumb">
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/main.html">User Guide</a></li>
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/configuration/main.html">Configuration</a></li>
+
+
+ <li class="active">Write Proxy Configuration</li>
+ </ol>
+
+ <div class="text">
+ <!-- Content -->
+ <div class="contents topic" id="write-proxy-configuration">
+<p class="topic-title first">Write Proxy Configuration</p>
+<ul class="simple">
+<li><a class="reference internal" href="#id1" id="id2">Write Proxy Configuration</a><ul>
+<li><a class="reference internal" href="#server-configuration-settings" id="id3">Server Configuration Settings</a><ul>
+<li><a class="reference internal" href="#rate-limit-settings" id="id4">Rate Limit Settings</a></li>
+<li><a class="reference internal" href="#stream-limit-settings" id="id5">Stream Limit Settings</a></li>
+</ul>
+</li>
+</ul>
+</li>
+</ul>
+</div>
+<div class="section" id="id1">
+<h2><a class="toc-backref" href="#id2">Write Proxy Configuration</a></h2>
+<p>This section describes the configuration settings used by DistributedLog Write Proxy.</p>
+<p>All the server related settings are managed in <cite>ServerConfiguration</cite>. Similar as <cite>DistributedLogConfiguration</cite>,
+it is also a properties based configuration, which extends from Apache commons <cite>CompositeConfiguration</cite>. All
+server related settings are in lower case and use <cite>'_'</cite> to concat words. For example, <cite>server_region_id</cite> means
+the region id used by the write proxy server.</p>
+<div class="section" id="server-configuration-settings">
+<h3><a class="toc-backref" href="#id3">Server Configuration Settings</a></h3>
+<ul class="simple">
+<li><em>server_dlsn_version</em>: The version of serialized format of DLSN. The default value is 1. It is not recommended to change it.</li>
+<li><em>server_region_id</em>: The region id used by the server to instantiate a DL namespace. The default value is <cite>LOCAL</cite>.</li>
+<li><em>server_port</em>: The listen port of the write proxy. The default value is 0.</li>
+<li><em>server_shard</em>: The shard id used by the server to identify itself. It is optional but recommended to set. For example, if
+the write proxy is running in <cite>Apache Aurora</cite>, you could use the instance id as the shard id. The default value is -1 (unset).</li>
+<li><em>server_threads</em>: The number of threads for the executor of this server. The default value is the available processors.</li>
+<li><em>server_enable_perstream_stat</em>: The flag to enable per stream stat in write proxy. It is different from <cite>enablePerStreamStat</cite>
+in core library. The setting here is controlling exposing the per stream stat exposed by write proxy, while <cite>enablePerStreamStat</cite>
+is to control expose the per stream stat exposed by the core library. It is enabled by default.</li>
+<li><em>server_graceful_shutdown_period_ms</em>: The graceful shutdown period in milliseconds. The default value is 0.</li>
+<li><em>server_service_timeout_ms</em>: The timeout period for the execution of a stream operation in write proxy. If it is positive,
+write proxy will timeout requests if they are taking longer time than the threshold. Otherwise, the timeout feature is disabled.
+By default, it is 0 (disabled).</li>
+<li><em>server_stream_probation_timeout_ms</em>: The time period that a stream should be kept in cache in probationary state after service
+timeout, in order to prevent ownership reacquiring. The unit is milliseconds. The default value is 5 minutes.</li>
+<li><em>stream_partition_converter_class</em>: The stream-to-partition convert class. The converter is used to group streams together, which
+these streams can apply same <cite>per-stream</cite> configuration settings or same other constraints. By default, it is an
+<cite>IdentityStreamPartitionConverter</cite> which doesn't group any streams.</li>
+</ul>
+<div class="section" id="rate-limit-settings">
+<h4><a class="toc-backref" href="#id4">Rate Limit Settings</a></h4>
+<p>This section describes the rate limit settings per write proxy.</p>
+<p>All the rate limit settings have both <cite>soft</cite> and <cite>hard</cite> thresholds. If the throughput goes above <cite>soft</cite> limit,
+the requests won't be rejected but just logging in the stat. But if the throughput goes above <cite>hard</cite> limit,
+the requests would be rejected immediately.</p>
+<p>NOTE: <cite>bps</cite> stands for <cite>bytes per second</cite>, while <cite>rps</cite> stands for <cite>requests per second</cite>.</p>
+<ul class="simple">
+<li><em>bpsSoftServiceLimit</em>: The soft limit for bps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+<li><em>bpsHardServiceLimit</em>: The hard limit for bps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+<li><em>rpsSoftServiceLimit</em>: The soft limit for rps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+<li><em>rpsHardServiceLimit</em>: The hard limit for rps. Setting it to 0 or negative value will disable this feature.
+By default it is disabled.</li>
+</ul>
+<p>There are two additional rate limiting settings that related to stream acquisitions.</p>
+<ul class="simple">
+<li><em>rpsStreamAcquireServiceLimit</em>: The rate limit for rps. When the rps goes above this threshold, the write proxy
+will stop accepting serving new streams.</li>
+<li><em>bpsStreamAcquireServiceLimit</em>: The rate limit for bps. When the bps goes above this threshold, the write proxy
+will stop accepting serving new streams.</li>
+</ul>
+</div>
+<div class="section" id="stream-limit-settings">
+<h4><a class="toc-backref" href="#id5">Stream Limit Settings</a></h4>
+<p>This section describes the stream limit settings per write proxy. They are the constraints that each write proxy
+will apply when deciding whether to own given streams.</p>
+<ul class="simple">
+<li><em>maxAcquiredPartitionsPerProxy</em>: The maximum number of partitions per stream that a write proxy is allowed to
+serve. Setting it to 0 or negative value will disable this feature. By default it is unlimited.</li>
+<li><em>maxCachedPartitionsPerProxy</em>: The maximum number of partitions per stream that a write proxy is allowed to cache.
+Setting it to 0 or negative value will disable this feature. By default it is unlimited.</li>
+</ul>
+</div>
+</div>
+</div>
+
+
+ </div>
+ </div>
+</div>
+
+
+
+ </div>
+
+
+ <hr>
+ <div class="row">
+ <div class="col-xs-12">
+ <footer>
+ <p class="text-center">© Copyright 2016
+ <a href="http://www.apache.org">The Apache Software Foundation.</a> All Rights Reserved.
+ </p>
+ <p class="text-center">
+ <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a>
+ </p>
+ </footer>
+ </div>
+ </div>
+ <!-- container div end -->
+</div>
+
+
+ <script>
+ (function () {
+ 'use strict';
+ anchors.options.placement = 'right';
+ anchors.add();
+ })();
+</script>
+
+ </body>
+
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/ef7245e8/content/docs/0.4.0-incubating/user_guide/considerations/main.html
----------------------------------------------------------------------
diff --git a/content/docs/0.4.0-incubating/user_guide/considerations/main.html b/content/docs/0.4.0-incubating/user_guide/considerations/main.html
new file mode 100644
index 0000000..0f25aea
--- /dev/null
+++ b/content/docs/0.4.0-incubating/user_guide/considerations/main.html
@@ -0,0 +1,658 @@
+<!DOCTYPE html>
+<html lang="en">
+
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <title>Apache DistributedLog (incubating)</title>
+ <meta name="description" content="Apache DistributedLog is an high performance replicated log.
+">
+
+ <link rel="stylesheet" href="/docs/0.4.0-incubating/styles/site.css">
+ <link rel="stylesheet" href="/docs/0.4.0-incubating/css/theme.css">
+ <!-- JQuery -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js"></script>
+ <script src="/docs/0.4.0-incubating/js/bootstrap.min.js"></script>
+ <link rel="canonical" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/user_guide/considerations/main.html" data-proofer-ignore>
+ <link rel="alternate" type="application/rss+xml" title="Apache DistributedLog (incubating)" href="http://distributedlog.incubator.apache.org/docs/0.4.0-incubating/feed.xml">
+ <!-- Font Awesome -->
+ <script src="//cdnjs.cloudflare.com/ajax/libs/anchor-js/3.2.0/anchor.min.js"></script>
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-83870961-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+ <!-- End Google Analytics -->
+ <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico">
+</head>
+
+
+ <body role="document">
+
+
+<nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <a href="/" class="navbar-brand" >
+ <img alt="Brand" style="height: 28px" src="/docs/0.4.0-incubating/images/distributedlog_logo_navbar.png">
+ </a>
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ </div>
+ <div id="navbar" class="navbar-collapse collapse">
+ <ul class="nav navbar-nav">
+ <!-- Overview -->
+ <li><a href="/docs/0.4.0-incubating/">V0.4.0</a></li>
+ <!-- Concepts -->
+ <li><a href="/docs/0.4.0-incubating/basics/introduction">Concepts</a></li>
+ <!-- Quick Start -->
+ <li>
+ <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Start<span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/building.html">
+ Build DistributedLog from Source
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/download.html">
+ Download Releases
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Quickstart</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/start/quickstart.html">
+ Setup & Run Example
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-1.html">
+ API - Write Records (via core library)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-2.html">
+ API - Write Records (via write proxy)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/basic-5.html">
+ API - Read Records
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Deployment</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/cluster.html">
+ Cluster Setup
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/global-cluster.html">
+ Global Cluster Setup
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/deployment/docker.html">
+ Docker
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- API -->
+ <li>
+ <a href="/docs/0.4.0-incubating/start" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">API<span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="/docs/0.4.0-incubating/api/java">Java</a></li>
+ </ul>
+ </li>
+ <!-- User Guide -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">User Guide<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/basics/introduction.html">
+ Introduction
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html">
+ Considerations
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html">
+ Architecture
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/main.html">
+ API
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html">
+ Configuration
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/design/main.html">
+ Detail Design
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html">
+ Global Replicated Log
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html">
+ Implementation
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/main.html">
+ References
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- Admin Guide -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Admin Guide<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/0.4.0-incubating/deployment/cluster">Cluster Setup</a></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/operations.html">
+ Operations
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/performance.html">
+ Performance Tuning
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/loadtest.html">
+ Load Test
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/hardware.html">
+ Hardware
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/monitoring.html">
+ Monitoring
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/zookeeper.html">
+ ZooKeeper
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/admin_guide/bookkeeper.html">
+ BookKeeper
+ </a>
+ </li>
+
+ </ul>
+ </li>
+ <!-- Tutorials -->
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Tutorials<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li class="dropdown-header"><strong>Basic</strong></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-1">Write Records (via Core Library)</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-2">Write Records (via Write Proxy)</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-3">Write Records to multiple streams</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-4">Atomic Write Records</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-5">Tailing Read Records</a></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/basic-6">Rewind Read Records</a></li>
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Messaging</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-1.html">
+ Write records to partitioned streams
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-2.html">
+ Write records to multiple streams (load balancer)
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-3.html">
+ At-least-once Processing
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-4.html">
+ Exact-Once Processing
+ </a>
+ </li>
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/messaging-5.html">
+ Implement a kafka-like pub/sub system
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Replicated State Machines</strong></li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/tutorials/replicatedstatemachines.html">
+ Build replicated state machines
+ </a>
+ </li>
+
+ <li role="separator" class="divider"></li>
+ <li class="dropdown-header"><strong>Analytics</strong></li>
+ <li><a href="/docs/0.4.0-incubating/tutorials/analytics-mapreduce">Process log streams using MapReduce</a></li>
+ </ul>
+ </li>
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+</nav>
+
+
+<link rel="stylesheet" href="">
+
+
+ <div class="container" role="main">
+
+ <div class="row">
+
+ <!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+
+<div class="row">
+ <!-- Sub Navigation -->
+ <div class="col-sm-3">
+ <ul id="sub-nav">
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/main.html" class="">User Guide</a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/basics/introduction.html" class="">
+ Introduction
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/considerations/main.html" class="active">
+ Considerations
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/architecture/main.html" class="">
+ Architecture
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/main.html" class="">
+ API
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/core.html" class="active">
+ Core Library API
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/proxy.html" class="active">
+ Proxy Client API
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/api/practice.html" class="active">
+ Best Practise
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/main.html" class="">
+ Configuration
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/core.html" class="active">
+ Core Library Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/proxy.html" class="active">
+ Write Proxy Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/client.html" class="active">
+ Client Configuration
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/configuration/perlog.html" class="active">
+ Per Stream Configuration
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/design/main.html" class="">
+ Detail Design
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/globalreplicatedlog/main.html" class="">
+ Global Replicated Log
+ </a>
+
+ <ul>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/main.html" class="">
+ Implementation
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/implementation/storage.html" class="active">
+ Storage
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/main.html" class="">
+ References
+ </a>
+
+ <ul>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/metrics.html" class="active">
+ Metrics
+ </a>
+ </li>
+
+
+ <li>
+ <a href="/docs/0.4.0-incubating/user_guide/references/features.html" class="active">
+ Available Features
+ </a>
+ </li>
+
+ </ul>
+
+ </li>
+
+ </ul>
+
+ </li>
+
+ </ul>
+ </div>
+ <!-- Main -->
+ <div class="col-sm-9">
+ <!-- Top anchor -->
+ <a href="#top"></a>
+
+ <!-- Breadcrumbs above the main heading -->
+ <ol class="breadcrumb">
+
+
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/0.4.0-incubating/user_guide/main.html">User Guide</a></li>
+
+
+ <li class="active">Considerations</li>
+ </ol>
+
+ <div class="text">
+ <!-- Content -->
+ <p>As different applications have different requirements, we've carefully considered the capabilities
+that should be included in DistributedLog leaving the rest up to the applications. These considerations are:</p>
+<div class="section" id="consistency-durability-and-ordering">
+<h2>Consistency, Durability and Ordering</h2>
+<p>The distributed systems literature commonly refers to two broad paradigms to use a log
+for building reliable replicated systems (Figure 1). The <cite>Pub-Sub</cite> paradigm usually
+refers to an active-active model where we keep a log of the incoming requests and each
+replica(reader) processes each request. While the <cite>Master-Slave</cite> paradigm elects one
+replica as the master to process requests as they arrive and log changes to its state.
+The other replicas referred to as slaves apply the state changes in the same order as
+the master, thereby being in sync and ready to take over the mastership if the current
+master fails. If the current master loses connectivity to the slaves due to a network
+partition, the slaves may elect a new master to continue forward progress. A fencing
+mechanism is necessary for the old master to discover that it has lost ownership and
+prevent it from modifying state after network connectivity is restored.</p>
+<div class="figure align-center">
+<img alt="../../images/pubsub.png" src="../../images/pubsub.png" style="width: 600px;" />
+<p class="caption">Figure 1. The uses of a log in distributed systems</p>
+</div>
+<p>These two different approaches indicate two different types of ordering requirements -
+<cite>Write Ordering</cite> and <cite>Read Ordering</cite>. <cite>Write ordering</cite> requires that all writes issued
+by the log writer be written in a strict order to the log, while <cite>read ordering</cite> only
+requires that any reader that reads the log stream should see the same record at any
+given position, the log records however may not appear in the same order that the writer
+wrote them. The replicated log service should be able to support both use cases.</p>
+</div>
+<div class="section" id="partitioning">
+<h2>Partitioning</h2>
+<p><cite>Partitioning</cite> (also known as sharding or bucketing) facilitates horizontal scale. The
+partitioning scheme depends on the characteristics of the application and is closely
+related to the ordering guarantees that the application requires. For example, distributed
+key/value store that uses DistributedLog as its transaction log, distributes the data into
+partitions each of which is a unit of consistency. Modifications within each partition are
+required to be strictly ordered. On the other hand, real-time analytics workloads don't
+require strict order, can use <em>round-robin</em> partitioning to evenly distribute the reads and
+writes across all partitions. It is therefore prudent to provide applications the flexibility
+to choose a suitable partitioning scheme.</p>
+</div>
+<div class="section" id="processing-semantics">
+<h2>Processing Semantics</h2>
+<p>Applications typically choose between <cite>at-least-once</cite> and <cite>exactly-once</cite> processing semantics.
+<cite>At-least-once</cite> processing guarantees that the application will process all the log records,
+however when the application resumes after failure, previously processed records may be
+re-processed if they have not been acknowledged. <cite>Exactly once</cite> processing is a stricter
+guarantee where applications must see the effect of processing each record exactly once.
+<cite>Exactly once</cite> semantics can be achieved by maintaining reader positions together with the
+application state and atomically updating both the reader position and the effects of the
+corresponding log records. For instance, for strongly consistent updates in a distributed
+key/value store the reader position must be persisted atomically with the changes applied
+from the corresponding log records. Upon restart from a failure, the reader resumes from the
+last persisted position thereby guaranteeing that each change is applied only once. With at
+least once processing guarantees the application can store reader positions in an external
+store and update it periodically. Upon restart the application will reprocess messages since
+the last updated reader position.</p>
+</div>
+
+
+ </div>
+ </div>
+</div>
+
+
+
+ </div>
+
+
+ <hr>
+ <div class="row">
+ <div class="col-xs-12">
+ <footer>
+ <p class="text-center">© Copyright 2016
+ <a href="http://www.apache.org">The Apache Software Foundation.</a> All Rights Reserved.
+ </p>
+ <p class="text-center">
+ <a href="/docs/0.4.0-incubating/feed.xml">RSS Feed</a>
+ </p>
+ </footer>
+ </div>
+ </div>
+ <!-- container div end -->
+</div>
+
+
+ <script>
+ (function () {
+ 'use strict';
+ anchors.options.placement = 'right';
+ anchors.add();
+ })();
+</script>
+
+ </body>
+
+</html>