You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by es...@apache.org on 2011/10/18 00:30:44 UTC
svn commit: r1185403 - in
/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink:
AvroSink.java LoggerSink.java NullSink.java
Author: esammer
Date: Mon Oct 17 22:30:43 2011
New Revision: 1185403
URL: http://svn.apache.org/viewvc?rev=1185403&view=rev
Log:
FLUME-787: Write javadoc for builtin sinks
Modified:
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/LoggerSink.java
incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/NullSink.java
Modified: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java?rev=1185403&r1=1185402&r2=1185403&view=diff
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java (original)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/AvroSink.java Mon Oct 17 22:30:43 2011
@@ -19,6 +19,7 @@ import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSink;
+import org.apache.flume.Sink;
import org.apache.flume.Transaction;
import org.apache.flume.conf.Configurable;
import org.apache.flume.source.avro.AvroFlumeEvent;
@@ -28,6 +29,69 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
+/**
+ * <p>
+ * A {@link Sink} implementation that can send events to an Avro server (such as
+ * Flume's <tt>AvroSource</tt>).
+ * </p>
+ * <p>
+ * This sink forms one half of Flume's tiered collection support. Events sent to
+ * this sink are turned into {@link AvroFlumeEvent}s and sent to the configured
+ * hostname / port pair using Avro's {@link NettyTransceiver}. The intent is
+ * that the destination is an instance of Flume's <tt>AvroSource</tt> which
+ * allows Flume to nodes to forward to other Flume nodes forming a tiered
+ * collection infrastructure. Of course, nothing prevents one from using this
+ * sink to speak to other custom built infrastructure that implements the same
+ * Avro protocol (specifically {@link AvroSourceProtocol}).
+ * </p>
+ * <p>
+ * Events are taken from the configured {@link Channel} in batches of the
+ * configured <tt>batch-size</tt>. The batch size has no theoretical limits
+ * although all events in the batch <b>must</b> fit in memory. Generally, larger
+ * batches are far more efficient, but introduce a slight delay (measured in
+ * millis) in delivery. The batch behavior is such that underruns (i.e. batches
+ * smaller than the configured batch size) are possible. This is a compromise
+ * made to maintain low latency of event delivery. If the channel returns a null
+ * event, meaning it is empty, the batch is immediately sent, regardless of
+ * size. Batch underruns are tracked in the metrics. Empty batches do not incur
+ * an RPC roundtrip.
+ * </p>
+ * <p>
+ * <b>Configuration options</b>
+ * </p>
+ * <table>
+ * <tr>
+ * <th>Parameter</th>
+ * <th>Description</th>
+ * <th>Unit / Type</th>
+ * <th>Default</th>
+ * </tr>
+ * <tr>
+ * <td><tt>hostname</tt></td>
+ * <td>The hostname to which events should be sent.</td>
+ * <td>Hostname or IP / String</td>
+ * <td>none (required)</td>
+ * </tr>
+ * <tr>
+ * <td><tt>port</tt></td>
+ * <td>The port to which events should be sent on <tt>hostname</tt>.</td>
+ * <td>TCP port / int</td>
+ * <td>none (required)</td>
+ * </tr>
+ * <tr>
+ * <td><tt>batch-size</tt></td>
+ * <td>The maximum number of events to send per RPC.</td>
+ * <td>events / int</td>
+ * <td>100</td>
+ * </tr>
+ * </table>
+ * <p>
+ * <b>Metrics</b>
+ * </p>
+ * <p>
+ * TODO
+ * </p>
+ */
public class AvroSink extends AbstractSink implements PollableSink,
Configurable {
Modified: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/LoggerSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/LoggerSink.java?rev=1185403&r1=1185402&r2=1185403&view=diff
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/LoggerSink.java (original)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/LoggerSink.java Mon Oct 17 22:30:43 2011
@@ -21,10 +21,33 @@ import org.apache.flume.Channel;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSink;
+import org.apache.flume.Sink;
import org.apache.flume.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * <p>
+ * A {@link Sink} implementation that logs all events received at the INFO level
+ * to the <tt>org.apache.flume.sink.LoggerSink</tt> logger.
+ * </p>
+ * <p>
+ * <b>WARNING:</b> Logging events can potentially introduce performance
+ * degradation.
+ * </p>
+ * <p>
+ * <b>Configuration options</b>
+ * </p>
+ * <p>
+ * <i>This sink has no configuration parameters.</i>
+ * </p>
+ * <p>
+ * <b>Metrics</b>
+ * </p>
+ * <p>
+ * TODO
+ * </p>
+ */
public class LoggerSink extends AbstractSink implements PollableSink {
private static final Logger logger = LoggerFactory
@@ -41,8 +64,11 @@ public class LoggerSink extends Abstract
event = channel.take();
if (event != null) {
- logger.info("Event: " + event);
- transaction.commit();
+ if (logger.isInfoEnabled()) {
+ logger.info("Event: " + event);
+ transaction.commit();
+ }
+
return Status.READY;
} else {
transaction.rollback();
Modified: incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/NullSink.java
URL: http://svn.apache.org/viewvc/incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/NullSink.java?rev=1185403&r1=1185402&r2=1185403&view=diff
==============================================================================
--- incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/NullSink.java (original)
+++ incubator/flume/branches/flume-728/flume-ng-core/src/main/java/org/apache/flume/sink/NullSink.java Mon Oct 17 22:30:43 2011
@@ -22,10 +22,29 @@ import org.apache.flume.CounterGroup;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSink;
+import org.apache.flume.Sink;
import org.apache.flume.Transaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * <p>
+ * A {@link Sink} implementation that simply discards all events it receives. A
+ * <tt>/dev/null</tt> for Flume.
+ * </p>
+ * <p>
+ * <b>Configuration options</b>
+ * </p>
+ * <p>
+ * <i>This sink has no configuration parameters.</i>
+ * </p>
+ * <p>
+ * <b>Metrics</b>
+ * </p>
+ * <p>
+ * TODO
+ * </p>
+ */
public class NullSink extends AbstractSink implements PollableSink {
private static final Logger logger = LoggerFactory.getLogger(NullSink.class);