You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by mp...@apache.org on 2012/09/13 02:34:11 UTC
git commit: FLUME-1572. Add batching support to FILE_ROLL sink.
Updated Branches:
refs/heads/trunk 960d03d8a -> 46659c715
FLUME-1572. Add batching support to FILE_ROLL sink.
(Hari Shreedharan via Mike Percy)
Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/46659c71
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/46659c71
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/46659c71
Branch: refs/heads/trunk
Commit: 46659c7156565778068c012f07301aab426d0728
Parents: 960d03d
Author: Mike Percy <mp...@apache.org>
Authored: Wed Sep 12 17:33:16 2012 -0700
Committer: Mike Percy <mp...@apache.org>
Committed: Wed Sep 12 17:33:16 2012 -0700
----------------------------------------------------------------------
.../org/apache/flume/sink/RollingFileSink.java | 47 ++++++++------
.../org/apache/flume/sink/TestRollingFileSink.java | 3 +
2 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flume/blob/46659c71/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java b/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
index e5e97ff..a94eea1 100644
--- a/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
+++ b/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
@@ -47,6 +47,9 @@ public class RollingFileSink extends AbstractSink implements Configurable {
private static final Logger logger = LoggerFactory
.getLogger(RollingFileSink.class);
private static final long defaultRollInterval = 30;
+ private static final int defaultBatchSize = 100;
+
+ private int batchSize = defaultBatchSize;
private File directory;
private long rollInterval;
@@ -90,6 +93,8 @@ public class RollingFileSink extends AbstractSink implements Configurable {
this.rollInterval = Long.parseLong(rollInterval);
}
+ batchSize = context.getInteger("sink.batchSize", defaultBatchSize);
+
this.directory = new File(directory);
}
@@ -175,30 +180,32 @@ public class RollingFileSink extends AbstractSink implements Configurable {
try {
transaction.begin();
- event = channel.take();
-
- if (event != null) {
- serializer.write(event);
-
- /*
- * FIXME: Feature: Rotate on size and time by checking bytes written and
- * setting shouldRotate = true if we're past a threshold.
- */
-
- /*
- * FIXME: Feature: Control flush interval based on time or number of
- * events. For now, we're super-conservative and flush on each write.
- */
- serializer.flush();
- outputStream.flush();
- } else {
- // No events found, request back-off semantics from runner
- result = Status.BACKOFF;
+ for (int i = 0; i < batchSize; i++) {
+ event = channel.take();
+ if (event != null) {
+ serializer.write(event);
+
+ /*
+ * FIXME: Feature: Rotate on size and time by checking bytes written and
+ * setting shouldRotate = true if we're past a threshold.
+ */
+
+ /*
+ * FIXME: Feature: Control flush interval based on time or number of
+ * events. For now, we're super-conservative and flush on each write.
+ */
+ } else {
+ // No events found, request back-off semantics from runner
+ result = Status.BACKOFF;
+ break;
+ }
}
+ serializer.flush();
+ outputStream.flush();
transaction.commit();
} catch (Exception ex) {
transaction.rollback();
- throw new EventDeliveryException("Failed to process event: " + event, ex);
+ throw new EventDeliveryException("Failed to process transaction", ex);
} finally {
transaction.close();
}
http://git-wip-us.apache.org/repos/asf/flume/blob/46659c71/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
----------------------------------------------------------------------
diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
index 10c9b82..07fa644 100644
--- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
+++ b/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
@@ -84,6 +84,7 @@ public class TestRollingFileSink {
context.put("sink.directory", tmpDir.getPath());
context.put("sink.rollInterval", "1");
+ context.put("sink.batchSize", "1");
Configurables.configure(sink, context);
@@ -131,6 +132,8 @@ public class TestRollingFileSink {
context.put("sink.directory", tmpDir.getPath());
context.put("sink.rollInterval", "0");
+ context.put("sink.batchSize", "1");
+
Configurables.configure(sink, context);
Re: git commit: FLUME-1572. Add batching support to FILE_ROLL
sink.
Posted by Hari Shreedharan <hs...@cloudera.com>.
Cherrypicked this to flume-1.3.0
Thanks,
Hari
--
Hari Shreedharan
On Wednesday, September 12, 2012 at 5:34 PM, mpercy@apache.org wrote:
> Updated Branches:
> refs/heads/trunk 960d03d8a -> 46659c715
>
>
> FLUME-1572. Add batching support to FILE_ROLL sink.
>
> (Hari Shreedharan via Mike Percy)
>
>
> Project: http://git-wip-us.apache.org/repos/asf/flume/repo
> Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/46659c71
> Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/46659c71
> Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/46659c71
>
> Branch: refs/heads/trunk
> Commit: 46659c7156565778068c012f07301aab426d0728
> Parents: 960d03d
> Author: Mike Percy <mpercy@apache.org (mailto:mpercy@apache.org)>
> Authored: Wed Sep 12 17:33:16 2012 -0700
> Committer: Mike Percy <mpercy@apache.org (mailto:mpercy@apache.org)>
> Committed: Wed Sep 12 17:33:16 2012 -0700
>
> ----------------------------------------------------------------------
> .../org/apache/flume/sink/RollingFileSink.java | 47 ++++++++------
> .../org/apache/flume/sink/TestRollingFileSink.java | 3 +
> 2 files changed, 30 insertions(+), 20 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/flume/blob/46659c71/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
> ----------------------------------------------------------------------
> diff --git a/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java b/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
> index e5e97ff..a94eea1 100644
> --- a/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
> +++ b/flume-ng-core/src/main/java/org/apache/flume/sink/RollingFileSink.java
> @@ -47,6 +47,9 @@ public class RollingFileSink extends AbstractSink implements Configurable {
> private static final Logger logger = LoggerFactory
> .getLogger(RollingFileSink.class);
> private static final long defaultRollInterval = 30;
> + private static final int defaultBatchSize = 100;
> +
> + private int batchSize = defaultBatchSize;
>
> private File directory;
> private long rollInterval;
> @@ -90,6 +93,8 @@ public class RollingFileSink extends AbstractSink implements Configurable {
> this.rollInterval = Long.parseLong(rollInterval);
> }
>
> + batchSize = context.getInteger("sink.batchSize", defaultBatchSize);
> +
> this.directory = new File(directory);
> }
>
> @@ -175,30 +180,32 @@ public class RollingFileSink extends AbstractSink implements Configurable {
>
> try {
> transaction.begin();
> - event = channel.take();
> -
> - if (event != null) {
> - serializer.write(event);
> -
> - /*
> - * FIXME: Feature: Rotate on size and time by checking bytes written and
> - * setting shouldRotate = true if we're past a threshold.
> - */
> -
> - /*
> - * FIXME: Feature: Control flush interval based on time or number of
> - * events. For now, we're super-conservative and flush on each write.
> - */
> - serializer.flush();
> - outputStream.flush();
> - } else {
> - // No events found, request back-off semantics from runner
> - result = Status.BACKOFF;
> + for (int i = 0; i < batchSize; i++) {
> + event = channel.take();
> + if (event != null) {
> + serializer.write(event);
> +
> + /*
> + * FIXME: Feature: Rotate on size and time by checking bytes written and
> + * setting shouldRotate = true if we're past a threshold.
> + */
> +
> + /*
> + * FIXME: Feature: Control flush interval based on time or number of
> + * events. For now, we're super-conservative and flush on each write.
> + */
> + } else {
> + // No events found, request back-off semantics from runner
> + result = Status.BACKOFF;
> + break;
> + }
> }
> + serializer.flush();
> + outputStream.flush();
> transaction.commit();
> } catch (Exception ex) {
> transaction.rollback();
> - throw new EventDeliveryException("Failed to process event: " + event, ex);
> + throw new EventDeliveryException("Failed to process transaction", ex);
> } finally {
> transaction.close();
> }
>
> http://git-wip-us.apache.org/repos/asf/flume/blob/46659c71/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
> ----------------------------------------------------------------------
> diff --git a/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java b/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
> index 10c9b82..07fa644 100644
> --- a/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
> +++ b/flume-ng-core/src/test/java/org/apache/flume/sink/TestRollingFileSink.java
> @@ -84,6 +84,7 @@ public class TestRollingFileSink {
>
> context.put("sink.directory", tmpDir.getPath());
> context.put("sink.rollInterval", "1");
> + context.put("sink.batchSize", "1");
>
> Configurables.configure(sink, context);
>
> @@ -131,6 +132,8 @@ public class TestRollingFileSink {
>
> context.put("sink.directory", tmpDir.getPath());
> context.put("sink.rollInterval", "0");
> + context.put("sink.batchSize", "1");
> +
>
> Configurables.configure(sink, context);
>
>
>