You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ju...@apache.org on 2013/04/29 05:50:25 UTC
svn commit: r1476886 - in /kafka/site: 07/includes/header.html 08/
08/api.html 08/configuration.html 08/design.html 08/includes/
08/includes/footer.html 08/includes/header.html 08/quickstart.html
08/tools.html includes/header.html
Author: junrao
Date: Mon Apr 29 03:50:25 2013
New Revision: 1476886
URL: http://svn.apache.org/r1476886
Log:
add 0.8 docs
Added:
kafka/site/08/
kafka/site/08/api.html
kafka/site/08/configuration.html
kafka/site/08/design.html
kafka/site/08/includes/
kafka/site/08/includes/footer.html
kafka/site/08/includes/header.html
kafka/site/08/quickstart.html
kafka/site/08/tools.html
Modified:
kafka/site/07/includes/header.html
kafka/site/includes/header.html
Modified: kafka/site/07/includes/header.html
URL: http://svn.apache.org/viewvc/kafka/site/07/includes/header.html?rev=1476886&r1=1476885&r2=1476886&view=diff
==============================================================================
--- kafka/site/07/includes/header.html (original)
+++ kafka/site/07/includes/header.html Mon Apr 29 03:50:25 2013
@@ -25,7 +25,7 @@
</div>
<div class="lsidebar">
<ul>
- <li>0.7</li>
+ <li>0.7
<ul>
<li><a href="quickstart.html">quickstart</a></li>
<li><a href="design.html">design</a></li>
@@ -34,6 +34,7 @@
<li><a href="configuration.html">configuration</a></li>
<li><a href="performance.html">performance</a></li>
</ul>
+ </li>
</ul>
</div>
<div class='content'>
Added: kafka/site/08/api.html
URL: http://svn.apache.org/viewvc/kafka/site/08/api.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/api.html (added)
+++ kafka/site/08/api.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,151 @@
+<!--#include virtual="includes/header.html" -->
+
+<h3>Producer Api</h3>
+<pre>
+/**
+ * V: type of the message
+ * K: type of the optional key assciated with the message
+ */
+class kafka.javaapi.producer.Producer<K,V>
+{
+ public Producer(ProducerConfig config);
+
+ /**
+ * Sends the data to a single topic, partitioned by key, using either the
+ * synchronous or the asynchronous producer
+ * @param message the producer data object that encapsulates the topic, key and message data
+ */
+ public void send(KeyedMessage<K,V> message);
+
+ /**
+ * Use this API to send data to multiple topics
+ * @param messages list of producer data objects that encapsulate the topic, key and message data
+ */
+ public void send(List<KeyedMessage<K,V>> messages);
+
+ /**
+ * Close API to close the producer pool connections to all Kafka brokers.
+ */
+ public void close();
+}
+
+</pre>
+You can follow
+<a href="https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example" title="Kafka 0.8 producer example">this</a> to learn how to use the producer api.
+
+<h3>High level Consumer Api</h3>
+<pre>
+class Consumer {
+ /**
+ * Create a ConsumerConnector
+ *
+ * @param config at the minimum, need to specify the groupid of the consumer and the zookeeper
+ * connection string zookeeper.connect.
+ */
+ public static kafka.javaapi.consumer.ConsumerConnector createJavaConsumerConnector(config: ConsumerConfig);
+}
+
+/**
+ * V: type of the message
+ * K: type of the optional key assciated with the message
+ */
+public interface kafka.javaapi.consumer.ConsumerConnector {
+ /**
+ * Create a list of message streams of type T for each topic.
+ *
+ * @param topicCountMap a map of (topic, #streams) pair
+ * @param decoder a decoder that converts from Message to T
+ * @return a map of (topic, list of KafkaStream) pairs.
+ * The number of items in the list is #streams. Each stream supports
+ * an iterator over message/metadata pairs.
+ */
+ public <K,V> Map<String, List<KafkaStream<K,V>>>
+ createMessageStreams(Map<String, Integer> topicCountMap, Decoder<K> keyDecoder, Decoder<V> valueDecoder);
+
+ /**
+ * Create a list of message streams of type T for each topic, using the default decoder.
+ */
+ public Map<String, List<KafkaStream<byte[], byte[]>>> createMessageStreams(Map<String, Integer> topicCountMap);
+
+ /**
+ * Create a list of message streams for topics matching a wildcard.
+ *
+ * @param topicFilter a TopicFilter that specifies which topics to
+ * subscribe to (encapsulates a whitelist or a blacklist).
+ * @param numStreams the number of message streams to return.
+ * @param keyDecoder a decoder that decodes the message key
+ * @param valueDecoder a decoder that decodes the message itself
+ * @return a list of KafkaStream. Each stream supports an
+ * iterator over its MessageAndMetadata elements.
+ */
+ public <K,V> List<KafkaStream<K,V>>
+ createMessageStreamsByFilter(TopicFilter topicFilter, int numStreams, Decoder<K> keyDecoder, Decoder<V> valueDecoder);
+
+ /**
+ * Create a list of message streams for topics matching a wildcard, using the default decoder.
+ */
+ public List<KafkaStream<byte[], byte[]>> createMessageStreamsByFilter(TopicFilter topicFilter, int numStreams);
+
+ /**
+ * Commit the offsets of all topic/partitions connected by this connector.
+ */
+ public void commitOffsets();
+
+ /**
+ * Shut down the connector
+ */
+ public void shutdown();
+}
+
+</pre>
+You can follow
+<a href="" title="Kafka 0.8 consumer example">this</a> to learn how to use the high level consumer api.
+
+<h3>SimpleConsumer Api</h3>
+<pre>
+class kafka.javaapi.consumer.SimpleConsumer {
+ /**
+ * Fetch a set of messages from a topic.
+ *
+ * @param request specifies the topic name, topic partition, starting byte offset, maximum bytes to be fetched.
+ * @return a set of fetched messages
+ */
+ public FetchResponse fetch(request: kafka.javaapi.FetchRequest);
+
+ /**
+ * Fetch metadata for a sequence of topics.
+ *
+ * @param request specifies the versionId, clientId, sequence of topics.
+ * @return metadata for each topic in the request.
+ */
+ public kafka.javaapi.TopicMetadataResponse send(request: kafka.javaapi.TopicMetadataRequest);
+
+ /**
+ * Get a list of valid offsets (up to maxSize) before the given time.
+ *
+ * @param request a [[kafka.javaapi.OffsetRequest]] object.
+ * @return a [[kafka.javaapi.OffsetResponse]] object.
+ */
+ public kafka.javaapi.OffsetResponse getOffsetsBefore(request: OffsetRequest);
+
+ /**
+ * Close the SimpleConsumer.
+ */
+ public void close();
+}
+</pre>
+For most applications, the high level consumer Api is good enough. Some applications want features not exposed to the high level consumer yet (e.g., set initial offset when restarting the consumer). They can instead use our low level SimpleConsumer Api. The logic will be a bit more complicated and you can follow the examples in
+<a href="https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+SimpleConsumer+Example" title="Kafka 0.8 SimpleConsumer example">here</a>.
+
+<h3>Kafka Hadoop Consumer</h3>
+
+<p>
+Providing a horizontally scalable solution for aggregating and loading data into Hadoop was one of our basic use cases. To support this use case, we provide a Hadoop-based consumer which spawns off many map tasks to pull data from the Kafka cluster in parallel. This provides extremely fast pull-based Hadoop data load capabilities (we were able to fully saturate the network with only a handful of Kafka servers).
+</p>
+
+<p>
+Usage information on the hadoop consumer can be found <a href="https://github.com/linkedin/camus/tree/camus-kafka-0.8/">here</a>.
+</p>
+
+<!--#include virtual="includes/footer.html" -->
+
Added: kafka/site/08/configuration.html
URL: http://svn.apache.org/viewvc/kafka/site/08/configuration.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/configuration.html (added)
+++ kafka/site/08/configuration.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,512 @@
+<!--#include virtual="includes/header.html" -->
+
+<h2> Configuration </h2>
+
+<h3> Important configuration properties for Kafka broker are: </h3>
+<ul>
+ <li><code>broker.id</code>
+ <li><code>log.dirs</code>
+ <li><code>zookeeper.connect</code>
+</ul>
+
+<p>More details about server configuration can be found in the scala class <code>kafka.server.KafkaConfig</code>.</p>
+
+<table class="data-table">
+<tbody><tr>
+ <th>Property</th>
+ <th>Default</th>
+ <th>Description</th>
+ </tr>
+ <tr>
+ <td>broker.id</td>
+ <td></td>
+ <td>Each broker is uniquely identified by a non-negative integer id. This id serves as the brokers "name", and allows the broker to be moved to a different host/port without confusing consumers.</td>
+ </tr>
+ <tr>
+ <td>log.dirs</td>
+ <td>"/tmp/kafka-logs"</td>
+ <td>The directories in which the log data is kept</td>
+ </tr>
+ <tr>
+ <td>zookeeper.connect</td>
+ <td>null</td>
+ <td>Specifies the zookeeper connection string in the form hostname:port/chroot. Here the chroot is a base directory which is prepended to all path operations (this effectively namespaces all kafka znodes to allow sharing with other applications on the same zookeeper cluster).</td>
+ </tr>
+ <tr>
+ <td>message.max.bytes</td>
+ <td>
+ <span>1000000</span>
+ </td>
+ <td>The maximum size of a message that the server can receive</td>
+ </tr>
+ <tr>
+ <td>num.network.threads</td>
+ <td>3</td>
+ <td>The number of network threads that the server uses for handling network requests</td>
+ </tr>
+ <tr>
+ <td>num.io.threads</td>
+ <td>8</td>
+ <td>The number of io threads that the server uses for carrying out network requests</td>
+ </tr>
+ <tr>
+ <td>queued.max.requests</td>
+ <td>500</td>
+ <td>The number of queued requests allowed before blocking the network threads</td>
+ </tr>
+ <tr>
+ <td>port</td>
+ <td>6667</td>
+ <td>The port to listen and accept connections on</td>
+ </tr>
+ <tr>
+ <td>host.name</td>
+ <td>null</td>
+ <td>
+ <p>Hostname of broker. If this is set, it will only bind to this address. If this is not set, it will bind to all interfaces, and publish one to ZK</p>
+ </td>
+ </tr>
+ <tr>
+ <td>socket.send.buffer.bytes</td>
+ <td>100 * 1024</td>
+ <td>The SO_SNDBUFF buffer of the socket sever sockets</td>
+ </tr>
+ <tr>
+ <td>socket.receive.buffer.bytes</td>
+ <td>100 * 1024</td>
+ <td>The SO_RCVBUFF buffer of the socket sever sockets</td>
+ </tr>
+ <tr>
+ <td>socket.request.max.bytes</td>
+ <td>100 * 1024 * 1024</td>
+ <td>The maximum number of bytes in a socket request</td>
+ </tr>
+ <tr>
+ <td>num.partitions</td>
+ <td>1</td>
+ <td>The default number of log partitions per topic</td>
+ </tr>
+ <tr>
+ <td>log.segment.bytes</td>
+ <td>1024 * 1024 * 1024</td>
+ <td>The maximum size of a single log file</td>
+ </tr>
+ <tr>
+ <td>log.segment.bytes.per.topic</td>
+ <td>""</td>
+ <td>The maximum size of a single log file for some specific topic</td>
+ </tr>
+ <tr>
+ <td>log.roll.hours</td>
+ <td>24 * 7</td>
+ <td>The maximum time before a new log segment is rolled out</td>
+ </tr>
+ <tr>
+ <td>log.roll.hours.per.topic</td>
+ <td>""</td>
+ <td>The number of hours before rolling out a new log segment for some specific topic</td>
+ </tr>
+ <tr>
+ <td>log.retention.hours</td>
+ <td>24 * 7</td>
+ <td>The number of hours to keep a log file before deleting it</td>
+ </tr>
+ <tr>
+ <td>log.retention.hours.per.topic</td>
+ <td>""</td>
+ <td>The number of hours to keep a log file before deleting it for some specific topic</td>
+ </tr>
+ <tr>
+ <td>log.retention.bytes</td>
+ <td>-1</td>
+ <td>The maximum size of the log before deleting it</td>
+ </tr>
+ <tr>
+ <td>log.retention.bytes.per.topic</td>
+ <td>""</td>
+ <td>The maximum size of the log for some specific topic before deleting it</td>
+ </tr>
+ <tr>
+ <td>log.cleanup.interval.mins</td>
+ <td>10</td>
+ <td>The frequency in minutes that the log cleaner checks whether any log is eligible for deletion</td>
+ </tr>
+ <tr>
+ <td>log.index.size.max.bytes</td>
+ <td>10 * 1024 * 1024</td>
+ <td>The maximum size in bytes of the offset index</td>
+ </tr>
+ <tr>
+ <td>log.index.interval.bytes</td>
+ <td>4096</td>
+ <td>The interval with which we add an entry to the offset index</td>
+ </tr>
+ <tr>
+ <td>log.flush.interval.messages</td>
+ <td>10000</td>
+ <td>The number of messages accumulated on a log partition before messages are flushed to disk</td>
+ </tr>
+ <tr>
+ <td>log.flush.interval.ms.per.topic</td>
+ <td>""</td>
+ <td>The maximum time in ms that a message in selected topics is kept in memory before flushed to disk, e.g., topic1:3000,topic2:6000</td>
+ </tr>
+ <tr>
+ <td>log.flush.scheduler.interval.ms</td>
+ <td>3000</td>
+ <td>The frequency in ms that the log flusher checks whether any log needs to be flushed to disk</td>
+ </tr>
+ <tr>
+ <td>log.flush.interval.ms</td>
+ <td>${<span>log.flush.scheduler.interval.ms}</span>
+ </td>
+ <td>The maximum time in ms that a message in any topic is kept in memory before flushed to disk</td>
+ </tr>
+ <tr>
+ <td>auto.create.topics.enable</td>
+ <td>true</td>
+ <td>Enable auto creation of topic on the server</td>
+ </tr>
+ <tr>
+ <td>controller.socket.timeout.ms</td>
+ <td>30000</td>
+ <td>The socket timeout for controller-to-broker channels</td>
+ </tr>
+ <tr>
+ <td>controller.message.queue.size</td>
+ <td>10</td>
+ <td>The buffer size for controller-to-broker-channels</td>
+ </tr>
+ <tr>
+ <td>default.replication.factor</td>
+ <td>1</td>
+ <td>Default replication factors for automatically created topics</td>
+ </tr>
+ <tr>
+ <td>replica.lag.time.max.ms</td>
+ <td>10000</td>
+ <td>If a follower hasn't sent any fetch requests during this time, the leader will remove the follower from isr</td>
+ </tr>
+ <tr>
+ <td>replica.lag.max.messages</td>
+ <td>4000</td>
+ <td>If the lag in messages between a leader and a follower exceeds this number, the leader will remove the follower from isr</td>
+ </tr>
+ <tr>
+ <td>replica.socket.timeout.ms</td>
+ <td>30 * 1000</td>
+ <td>The socket timeout for network requests</td>
+ </tr>
+ <tr>
+ <td>replica.socket.receive.buffer.bytes</td>
+ <td>64 * 1024</td>
+ <td>The socket receive buffer for network requests</td>
+ </tr>
+ <tr>
+ <td>replica.fetch.max.bytes</td>
+ <td>1024 * 1024</td>
+ <td>The number of byes of messages to attempt to fetch</td>
+ </tr>
+ <tr>
+ <td>replica.fetch.wait.max.ms</td>
+ <td>500</td>
+ <td>Max wait time for each fetcher request issued by follower replicas</td>
+ </tr>
+ <tr>
+ <td>replica.fetch.min.bytes</td>
+ <td>1</td>
+ <td>Minimum bytes expected for each fetch response. If not enough bytes, wait up to replicaMaxWaitTimeMs</td>
+ </tr>
+ <tr>
+ <td>num.replica.fetchers</td>
+ <td>1</td>
+ <td>
+ <p>Number of fetcher threads used to replicate messages from a source broker. Increasing this value can increase the degree of I/O parallelism in the follower broker.</p>
+ </td>
+ </tr>
+ <tr>
+ <td>replica.high.watermark.checkpoint.interval.ms</td>
+ <td>5000</td>
+ <td>The frequency with which the high watermark is saved out to disk</td>
+ </tr>
+ <tr>
+ <td>fetch.purgatory.purge.interval.requests</td>
+ <td>10000</td>
+ <td>The purge interval (in number of requests) of the fetch request purgatory</td>
+ </tr>
+ <tr>
+ <td>producer.purgatory.purge.interval.requests</td>
+ <td>10000</td>
+ <td>The purge interval (in number of requests) of the producer request purgatory</td>
+ </tr>
+ <tr>
+ <td>zookeeper.session.timeout.ms</td>
+ <td>6000</td>
+ <td>Zookeeper session timeout</td>
+ </tr>
+ <tr>
+ <td>zookeeper.connection.timeout.ms</td>
+ <td>${zookeeper.session.timeout.ms}</td>
+ <td>The max time that the client waits to establish a connection to zookeeper</td>
+ </tr>
+ <tr>
+ <td>zookeeper.sync.time.ms</td>
+ <td>2000</td>
+ <td>How far a ZK follower can be behind a ZK leader</td>
+ </tr>
+</tbody></table>
+
+
+<h3> Important configuration properties for the high-level consumer are: </h3>
+<ul>
+ <li><code>group.id</code>
+ <li><code>zookeeper.connect</code>
+</ul>
+
+<p>More details about consumer configuration can be found in the scala class <code>kafka.consumer.ConsumerConfig</code>.</p>
+
+<table class="data-table">
+<tbody><tr>
+ <th>Property</th>
+ <th>Default</th>
+ <th>Description</th>
+</tr>
+ <tr>
+ <td>group.id</td>
+ <td colspan="1"></td>
+ <td>A string that uniquely identifies a set of consumers within the same consumer group</td>
+ </tr>
+ <tr>
+ <td>zookeeper.connect</td>
+ <td colspan="1">null</td>
+ <td>Specifies the zookeeper connection string in the form hostname:port/chroot. Here the chroot is a base directory which is prepended to all path operations (this effectively namespaces all kafka znodes to allow sharing with other applications on the same zookeeper cluster.</td>
+ </tr>
+ <tr>
+ <td>consumer.id</td>
+ <td colspan="1">null</td>
+ <td>
+ <p>Generated automatically if not set.</p>
+ </td>
+ </tr>
+ <tr>
+ <td>socket.timeout.ms</td>
+ <td colspan="1">30 * 1000</td>
+ <td>The socket timeout for network requests. The actual timeout set will be max.fetch.wait + socket.timeout.ms.</td>
+ </tr>
+ <tr>
+ <td>socket.receive.buffer.bytes</td>
+ <td colspan="1">64 * 1024</td>
+ <td>The socket receive buffer for network requests</td>
+ </tr>
+ <tr>
+ <td>fetch.message.max.bytes</td>
+ <td colspan="1">1024 * 1024</td>
+ <td>The number of byes of messages to attempt to fetch</td>
+ </tr>
+ <tr>
+ <td>auto.commit.enable</td>
+ <td colspan="1">true</td>
+ <td>If true, periodically commit to zookeeper the offset of messages already fetched by the consumer</td>
+ </tr>
+ <tr>
+ <td>auto.commit.interval.ms</td>
+ <td colspan="1">60 * 1000</td>
+ <td>The frequency in ms that the consumer offsets are committed to zookeeper</td>
+ </tr>
+ <tr>
+ <td>queued.max.messages</td>
+ <td colspan="1">10</td>
+ <td>Max number of messages buffered for consumption</td>
+ </tr>
+ <tr>
+ <td>rebalance.max.retries</td>
+ <td colspan="1">4</td>
+ <td>Max number of retries during rebalance</td>
+ </tr>
+ <tr>
+ <td>fetch.min.bytes</td>
+ <td colspan="1">1</td>
+ <td>The minimum amount of data the server should return for a fetch request. If insufficient data is available the request will block</td>
+ </tr>
+ <tr>
+ <td>fetch.wait.max.ms</td>
+ <td colspan="1">100</td>
+ <td>The maximum amount of time the server will block before answering the fetch request if there isn't sufficient data to immediately satisfy fetch.min.bytes</td>
+ </tr>
+ <tr>
+ <td>rebalance.backoff.ms</td>
+ <td>${zookeeper.sync.time.ms}</td>
+ <td>Backoff time between retries during rebalance</td>
+ </tr>
+ <tr>
+ <td>refresh.leader.backoff.ms</td>
+ <td colspan="1">200</td>
+ <td>Backoff time to refresh the leader of a partition after it loses the current leader</td>
+ </tr>
+ <tr>
+ <td>auto.offset.reset</td>
+ <td colspan="1">"smallest"</td>
+ <td>
+ <p>What to do if an offset is out of range:<br/>* smallest : automatically reset the offset to the smallest offset<br/>* largest : automatically reset the offset to the largest offset<br/>* anything else: throw exception to the consumer</p>
+ </td>
+ </tr>
+ <tr>
+ <td>consumer.timeout.ms</td>
+ <td colspan="1">-1</td>
+ <td>Throw a timeout exception to the consumer if no message is available for consumption after the specified interval</td>
+ </tr>
+ <tr>
+ <td>client.id</td>
+ <td colspan="1">${group.id}</td>
+ <td>Client id is specified by the kafka consumer client, used to distinguish different clients</td>
+ </tr>
+ <tr>
+ <td>zookeeper.session.timeout.ms </td>
+ <td colspan="1">6000</td>
+ <td>Zookeeper session timeout</td>
+ </tr>
+ <tr>
+ <td>zookeeper.connection.timeout.ms</td>
+ <td colspan="1">${<span>zookeeper.session.timeout.ms}</span>
+ </td>
+ <td>The max time that the client waits to establish a connection to zookeeper</td>
+ </tr>
+ <tr>
+ <td>zookeeper.sync.time.ms </td>
+ <td colspan="1">2000</td>
+ <td>How far a ZK follower can be behind a ZK leader</td>
+ </tr>
+</tbody></table>
+
+
+<h3> Important configuration properties for the producer are: </h3>
+<ul>
+ <li><code>metadata.broker.list</code>
+ <li><code>request.required.acks</code>
+ <li><code>producer.type</code>
+ <li><code>serializer.class</code>
+</ul>
+
+<p>More details about producer configuration can be found in the scala class <code>kafka.producer.ProducerConfig</code>.</p>
+
+<table class="data-table">
+<tbody><tr>
+ <th>Property</th>
+ <th>Default</th>
+ <th>Description</th>
+ </tr>
+ <tr>
+ <td>metadata.broker.list</td>
+ <td colspan="1"></td>
+ <td>
+ <p>This is for bootstrapping and the producer will only use it for getting metadata (topics, partitions and replicas). The socket connections for sending the actual data will be established based on the broker information returned in the metadata. The format is host1:port1,host2:port2, and the list can be a subset of brokers or a VIP pointing to a subset of brokers.</p>
+ </td>
+ </tr>
+ <tr>
+ <td>request.required.acks</td>
+ <td colspan="1">0</td>
+ <td>
+ <p>This value controls when the producer receives an acknowledgement from the broker. Typical values are (1) 0, which means that the producer never waits for an acknowledgement from the broker (the same behavior as 0.7); (2) 1, which means that the producer gets an acknowledgement after the leader replica has received the data; (3) -1, which means that the producer gets an acknowledgement after all in-sync replicas have received the data. The first option provides the lowest latency (no network delay), but the worst durabilty (some data loss when the leader replica fails). The second option provides lower latency (one network round trip) and better durability (few data loss when the leader replica fails). The last option provides low latency (two network round trips) and the best durability (no data loss as long as the number of failed brokers is less the replication factor of the topic).</p>
+ </td>
+ </tr>
+ <tr>
+ <td>producer.type</td>
+ <td colspan="1">"sync"</td>
+ <td>
+ <p>This parameter specifies whether the messages are sent asynchronously or not. Valid values are (1) async for asynchronous send and (2) sync for synchronous send.</p>
+ </td>
+ <tr>
+ <td>serializer.class</td>
+ <td colspan="1">"kafka.serializer.DefaultEncoder"</td>
+ <td>The serializer class for values</td>
+ </tr>
+ <tr>
+ <td>key.serializer.class</td>
+ <td colspan="1">${serializer.class}</td>
+ <td>The serializer class for keys (defaults to the same as for values)</td>
+ </tr>
+ <tr>
+ <td>partitioner.class</td>
+ <td colspan="1">"kafka.producer.DefaultPartitioner"</td>
+ <td>The partitioner class for partitioning events amongst sub-topics</td>
+ </tr>
+ </tr>
+ <tr>
+ <td>compression.codec</td>
+ <td colspan="1">"none"</td>
+ <td>
+ <p>This parameter allows you to specify the compression codec for all data generated by this producer. The default is NoCompressionCodec</p>
+ </td>
+ </tr>
+ <tr>
+ <td>compressed.topics</td>
+ <td colspan="1">null</td>
+ <td>
+ <p>This parameter allows you to set whether compression should be turned on for particular topics. If the compression codec is anything other than NoCompressionCodec, enable compression only for specified topics if any. If the list of compressed topics is empty, then enable the specified compression codec for all topics. If the compression codec is NoCompressionCodec, compression is disabled for all topics</p>
+ </td>
+ </tr>
+ <tr>
+ <td>message.send.max.retries</td>
+ <td colspan="1">3</td>
+ <td>
+ <p>The leader may be unavailable transiently, which can fail the sending of a message. This property specifies the number of retries when such failures occur.</p>
+ </td>
+ </tr>
+ <tr>
+ <td>retry.backoff.ms</td>
+ <td colspan="1">100</td>
+ <td>
+ <p>Before each retry, the producer refreshes the metadata of relevant topics. Since leader election takes a bit of time, this property specifies the amount of time that the producer waits before refreshing the metadata.</p>
+ </td>
+ </tr>
+ <tr>
+ <td>topic.metadata.refresh.interval.ms</td>
+ <td colspan="1">600* 1000</td>
+ <td>
+ <p>The producer generally refreshes the topic metadata from brokers when there is a failure (partition missing, leader not available...). It will also poll regularly (default: every 10min so 600000ms). If you set this to a negative value, metadata will only get refreshed on failure. If you set this to zero, the metadata will get refreshed after each message sent (not recommended). Important note: the refresh happen only AFTER the message is sent, so if the producer never sends a message the metadata is never refreshed</p>
+ </td>
+ </tr>
+ <tr>
+ <td>queue.buffering.max.ms</td>
+ <td colspan="1">5000</td>
+ <td>Maximum time, in milliseconds, for buffering data on the producer queue</td>
+ </tr>
+ <tr>
+ <td>queue.buffering.max.messages</td>
+ <td colspan="1">10000</td>
+ <td>The maximum size of the blocking queue for buffering on the producer</td>
+ </tr>
+ <tr>
+ <td>queue.enqueue.timeout.ms</td>
+ <td colspan="1">-1</td>
+ <td>
+ <p>Timeout for event enqueue:<br/> * 0: events will be enqueued immediately or dropped if the queue is full<br/> * -ve: enqueue will block indefinitely if the queue is full<br/> * +ve: enqueue will block up to this many milliseconds if the queue is full</p>
+ </td>
+ </tr>
+ <tr>
+ <td>batch.num.messages</td>
+ <td colspan="1">200</td>
+ <td>The number of messages batched at the producer</td>
+ </tr>
+ <tr>
+ <td>send.buffer.bytes</td>
+ <td colspan="1">100 * 1024</td>
+ <td>Socket write buffer size</td>
+ </tr>
+ <tr>
+ <td>client.id</td>
+ <td colspan="1">""</td>
+ <td>The client application sending the producer requests</td>
+ </tr>
+ <tr>
+ <td>request.timeout.ms</td>
+ <td colspan="1">1500</td>
+ <td>The ack timeout of the producer requests. Value must be non-negative and non-zero</td>
+ </tr>
+</tbody></table>
+
+
+ </div>
+
+
+</div></body></html>
Added: kafka/site/08/design.html
URL: http://svn.apache.org/viewvc/kafka/site/08/design.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/design.html (added)
+++ kafka/site/08/design.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,10 @@
+<!--#include virtual="includes/header.html" -->
+
+<h3> General design</h3>
+The motivation of Kafka and the general design can be found in the <a href="http://kafka.apache.org/07/design.html">design doc of 0.7</a>.
+
+<h3> Replication design</h3>
+The design of intra-cluster replication can be found in the <a href="http://www.slideshare.net/junrao/kafka-replication-apachecon2013">presentation</a> at ApacheCon 2013.
+
+<!--#include virtual="includes/footer.html" -->
+
Added: kafka/site/08/includes/footer.html
URL: http://svn.apache.org/viewvc/kafka/site/08/includes/footer.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/includes/footer.html (added)
+++ kafka/site/08/includes/footer.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,3 @@
+ </div>
+ </body>
+</html>
\ No newline at end of file
Added: kafka/site/08/includes/header.html
URL: http://svn.apache.org/viewvc/kafka/site/08/includes/header.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/includes/header.html (added)
+++ kafka/site/08/includes/header.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,39 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html xmlns:og="http://ogp.me/ns#">
+ <head>
+ <title>Apache Kafka</title>
+ <link rel='stylesheet' href='../styles.css' type='text/css'>
+ <link rel="icon" type="image/png" href="http://path/to/favicon.png"><meta name="keywords" content="keywords!">
+ <meta name="description" content="A high-throughput, distributed, publish-subscribe messaging system.">
+ <meta http-equiv='Content-Type' content='text/html;charset=utf-8' />
+ <meta property="og:title" value="Kafka" />
+ <meta property="og:image" value="" />
+ <meta property="og:description" value="A high-throughput, distributed, publish-subscribe messaging system." />
+ <meta property="og:site_name" value="sna-projects" />
+ <meta property="og:type" value="website" />
+ </head>
+ <body>
+ <div>
+ <div id="header" style="overflow: hidden">
+ <div>
+ <a href="http://www.apache.org">
+ <img style="float: right; margin: 5px" src="http://www.apache.org/images/feather-small.png"/>
+ </a>
+ <div class="title"><a href="../index.html">Apache Kafka</a></div>
+ <div class="subtitle">A high-throughput distributed messaging system.</div>
+ </div>
+ </div>
+ <div class="lsidebar">
+ <ul>
+ <li>0.8
+ <ul>
+ <li><a href="quickstart.html">quickstart</a></li>
+ <li><a href="design.html">design</a></li>
+ <li><a href="api.html">api docs</a></li>
+ <li><a href="configuration.html">configuration</a></li>
+ <li><a href="tools.html">tools</a></li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ <div class='content'>
Added: kafka/site/08/quickstart.html
URL: http://svn.apache.org/viewvc/kafka/site/08/quickstart.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/quickstart.html (added)
+++ kafka/site/08/quickstart.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,78 @@
+<!--#include virtual="includes/header.html" -->
+
+<h2>Quick Start</h3>
+
+<h3> Step 0: Clean up 0.7 data, if exists </h3>
+Since 0.8 is not backward compatible with 0.7.x, if you had an 0.7 installation already, you will need to delete all existing Zookeeper data and Kafka log data (unless, you use a different Zookeeper namespace and a new Kafka log directory).
+
+<h3> Step 1: Download the code </h3>
+
+<a href="../downloads.html" title="Kafka downloads">Download</a> a recent stable release.
+
+<pre>
+<b>> tar xzf kafka-<VERSION>.tgz</b>
+<b>> cd kafka-<VERSION></b>
+<b>> ./sbt update</b>
+<b>> ./sbt package</b>
+<b>> ./sbt assembly-package-dependency</b>
+</pre>
+
+<h3>Step 2: Start the server</h3>
+
+<p>
+First start the zookeeper server. You can use the convenience script packaged with kafka to get a quick-and-dirty single-node zookeeper instance.
+
+<pre>
+<b>> bin/zookeeper-server-start.sh config/zookeeper.properties</b>
+[2013-04-22 15:01:37,495] INFO Reading configuration from: config/zookeeper.properties (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
+...
+</pre>
+
+Now start the Kafka server:
+<pre>
+<b>> bin/kafka-server-start.sh config/server.properties</b>
+jkreps-mn-2:kafka-trunk jkreps$ bin/kafka-server-start.sh config/server.properties
+[2013-04-22 15:01:47,028] INFO Verifying properties (kafka.utils.VerifiableProperties)
+[2013-04-22 15:01:47,051] INFO Property socket.send.buffer.bytes is overridden to 1048576 (kafka.utils.VerifiableProperties)
+...
+</pre>
+
+<h3>Step 3: Send some messages</h3>
+
+Kafka comes with a command line client that will take input from standard in and send it out as messages to the Kafka cluster. By default each line will be sent as a separate message. The topic <i>test</i> is created automatically when messages are sent to it. Omitting logging you should see something like this:
+
+<pre>
+> <b>bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test</b>
+This is a message
+This is another message
+</pre>
+
+<h3>Step 4: Start a consumer</h3>
+
+Kafka also has a command line consumer that will dump out messages to standard out.
+
+<pre>
+<b>> bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning</b>
+This is a message
+This is another message
+</pre>
+<p>
+If you have each of the above commands running in a different terminal then you should now be able to type messages into the producer terminal and see them appear in the consumer terminal.
+</p>
+<p>
+Both of these command line tools have additional options. Running the command with no arguments will display usage information documenting them in more detail.
+</p>
+
+
+<h3>Setting up replication</h3>
+If you want to set up replication on more than one broker, you can follow
+<a href="https://cwiki.apache.org/confluence/display/KAFKA/Kafka+0.8+Quick+Start" title="Kafka 0.8 replication quickstart">this</a>.
+
+<h3>Build a release zip or tar file</h3>
+<pre>
+<b>> ./sbt release-zip</b>
+<b>> ./sbt release-tar</b>
+</pre>
+
+<!--#include virtual="includes/footer.html" -->
+
Added: kafka/site/08/tools.html
URL: http://svn.apache.org/viewvc/kafka/site/08/tools.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/tools.html (added)
+++ kafka/site/08/tools.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,7 @@
+<!--#include virtual="includes/header.html" -->
+
+<h3> Migrating data from 0.7 to 0.8</h3>
+Since 0.8 is not backward compatible with 0.7.x, we provide a tool for migrating data in an 0.7 cluster to an 0.8 cluster. Details of the tool can be found <a href="https://cwiki.apache.org/confluence/display/KAFKA/Migrating+from+0.7+to+0.8">here</a>.
+
+<!--#include virtual="includes/footer.html" -->
+
Modified: kafka/site/includes/header.html
URL: http://svn.apache.org/viewvc/kafka/site/includes/header.html?rev=1476886&r1=1476885&r2=1476886&view=diff
==============================================================================
--- kafka/site/includes/header.html (original)
+++ kafka/site/includes/header.html Mon Apr 29 03:50:25 2013
@@ -26,10 +26,12 @@
<div class="lsidebar">
<ul>
<li><a href="downloads.html">download</a></li>
- <li>releases</li>
+ <li>releases
<ul>
<li><a href="07/quickstart.html">0.7</a></li>
+ <li><a href="08/quickstart.html">0.8</a></li>
</ul>
+ </li>
<li><a href="https://cwiki.apache.org/confluence/display/KAFKA/Operations">operations</a></li>
<li><a href="faq.html">faq</a></li>
<li><a href="https://cwiki.apache.org/confluence/display/KAFKA">wiki</li>