You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by br...@apache.org on 2013/06/13 02:36:32 UTC

[1/3] FLUME-1586: File Channel should support verifying integrity of individual events.

Updated Branches:
  refs/heads/trunk 791f443fa -> c57ebd1d2


http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto b/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
index 1e668d2..25520e8 100644
--- a/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
+++ b/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
@@ -56,6 +56,7 @@ message TransactionEventHeader {
 
 message Put {
   required FlumeEvent event = 1;
+  optional sfixed64 checksum = 2;
 }
 
 message Take {

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
index 0f7d14d..25765b5 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
@@ -21,13 +21,17 @@ package org.apache.flume.channel.file;
 import static org.apache.flume.channel.file.TestUtils.*;
 import static org.fest.reflect.core.Reflection.*;
 
+import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
@@ -487,7 +491,7 @@ public class TestFileChannel extends TestFileChannelBase {
   public void testReferenceCounts() throws Exception {
     Map<String, String> overrides = Maps.newHashMap();
     overrides.put(FileChannelConfiguration.CHECKPOINT_INTERVAL, "10000");
-    overrides.put(FileChannelConfiguration.MAX_FILE_SIZE, "100");
+    overrides.put(FileChannelConfiguration.MAX_FILE_SIZE, "150");
     final FileChannel channel = createFileChannel(overrides);
     channel.start();
     putEvents(channel, "testing-reference-counting", 1, 15);
@@ -568,4 +572,42 @@ public class TestFileChannel extends TestFileChannelBase {
 
   }
 
+  @Test (expected = IllegalStateException.class)
+  public void testChannelDiesOnCorruptEvent() throws Exception {
+    final FileChannel channel = createFileChannel();
+    channel.start();
+    putEvents(channel,"test-corrupt-event",100,100);
+    for(File dataDir : dataDirs) {
+      File[] files = dataDir.listFiles(new FilenameFilter() {
+        @Override
+        public boolean accept(File dir, String name) {
+          if(!name.endsWith("meta") && !name.contains("lock")){
+            return true;
+          }
+          return false;
+        }
+      });
+      if (files != null && files.length > 0) {
+        for (int j = 0; j < files.length; j++) {
+          RandomAccessFile fileToCorrupt = new RandomAccessFile(files[0], "rw");
+          fileToCorrupt.seek(50);
+          fileToCorrupt.writeByte(234);
+          fileToCorrupt.close();
+        }
+      }
+    }
+    try {
+      consumeChannel(channel, true);
+    } catch (IllegalStateException ex) {
+      // The rollback call in takeEvents() in TestUtils will cause an
+      // IllegalArgumentException - and this should be tested to verify the
+      // channel is completely stopped.
+      Assert.assertTrue(ex.getMessage().contains("Log is closed"));
+      throw ex;
+    }
+    Assert.fail();
+
+
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
index 54978f8..8a5f8ad 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
@@ -74,7 +74,8 @@ public class TestLog {
    * not transactional so the commit is not required.
    */
   @Test
-  public void testPutGet() throws IOException, InterruptedException {
+  public void testPutGet()
+    throws IOException, InterruptedException, NoopRecordException {
     FlumeEvent eventIn = TestUtils.newPersistableEvent();
     long transactionID = ++this.transactionID;
     FlumeEventPointer eventPointer = log.put(transactionID, eventIn);
@@ -86,7 +87,8 @@ public class TestLog {
     Assert.assertArrayEquals(eventIn.getBody(), eventOut.getBody());
   }
   @Test
-  public void testRoll() throws IOException, InterruptedException {
+  public void testRoll()
+    throws IOException, InterruptedException, NoopRecordException {
     log.shutdownWorker();
     Thread.sleep(1000);
     for (int i = 0; i < 1000; i++) {
@@ -107,15 +109,16 @@ public class TestLog {
         }
       }
     }
-    // 78 (*2 for meta) files with TestLog.MAX_FILE_SIZE=1000
-    Assert.assertEquals(156, logCount);
+    // 93 (*2 for meta) files with TestLog.MAX_FILE_SIZE=1000
+    Assert.assertEquals(186, logCount);
   }
   /**
    * After replay of the log, we should find the event because the put
    * was committed
    */
   @Test
-  public void testPutCommit() throws IOException, InterruptedException {
+  public void testPutCommit()
+    throws IOException, InterruptedException, NoopRecordException {
     FlumeEvent eventIn = TestUtils.newPersistableEvent();
     long transactionID = ++this.transactionID;
     FlumeEventPointer eventPointerIn = log.put(transactionID, eventIn);
@@ -243,16 +246,16 @@ public class TestLog {
    */
   @Test
   public void testPutTakeRollbackLogReplayV1()
-      throws IOException, InterruptedException {
+    throws IOException, InterruptedException, NoopRecordException {
     doPutTakeRollback(true);
   }
   @Test
   public void testPutTakeRollbackLogReplayV2()
-      throws IOException, InterruptedException {
+    throws IOException, InterruptedException, NoopRecordException {
     doPutTakeRollback(false);
   }
   public void doPutTakeRollback(boolean useLogReplayV1)
-      throws IOException, InterruptedException {
+    throws IOException, InterruptedException, NoopRecordException {
     FlumeEvent eventIn = TestUtils.newPersistableEvent();
     long putTransactionID = ++transactionID;
     FlumeEventPointer eventPointerIn = log.put(putTransactionID, eventIn);
@@ -392,7 +395,7 @@ public class TestLog {
   }
   @Test
   public void testReplaySucceedsWithUnusedEmptyLogMetaDataNormalReplay()
-      throws IOException, InterruptedException {
+    throws IOException, InterruptedException, NoopRecordException {
     FlumeEvent eventIn = TestUtils.newPersistableEvent();
     long transactionID = ++this.transactionID;
     FlumeEventPointer eventPointer = log.put(transactionID, eventIn);
@@ -406,7 +409,7 @@ public class TestLog {
   }
   @Test
   public void testReplaySucceedsWithUnusedEmptyLogMetaDataFastReplay()
-      throws IOException, InterruptedException {
+    throws IOException, InterruptedException, NoopRecordException {
     FlumeEvent eventIn = TestUtils.newPersistableEvent();
     long transactionID = ++this.transactionID;
     FlumeEventPointer eventPointer = log.put(transactionID, eventIn);
@@ -422,7 +425,7 @@ public class TestLog {
   }
   public void doTestReplaySucceedsWithUnusedEmptyLogMetaData(FlumeEvent eventIn,
       FlumeEventPointer eventPointer) throws IOException,
-      InterruptedException {
+    InterruptedException, NoopRecordException {
     for (int i = 0; i < dataDirs.length; i++) {
       for(File logFile : LogUtils.getLogs(dataDirs[i])) {
         if(logFile.length() == 0L) {
@@ -461,7 +464,8 @@ public class TestLog {
   }
 
   private void takeAndVerify(FlumeEventPointer eventPointerIn,
-      FlumeEvent eventIn) throws IOException, InterruptedException {
+      FlumeEvent eventIn)
+    throws IOException, InterruptedException, NoopRecordException {
     FlumeEventQueue queue = log.getFlumeEventQueue();
     FlumeEventPointer eventPointerOut = queue.removeHead(0);
     Assert.assertNotNull(eventPointerOut);

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
index bef22ef..4da6ac1 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
@@ -21,12 +21,16 @@ package org.apache.flume.channel.file;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -98,6 +102,8 @@ public class TestLogFile {
     final List<Throwable> errors =
         Collections.synchronizedList(new ArrayList<Throwable>());
     ExecutorService executorService = Executors.newFixedThreadPool(10);
+    CompletionService<Void> completionService = new ExecutorCompletionService
+      <Void>(executorService);
     final LogFile.RandomReader logFileReader =
         LogFileFactory.getRandomReader(dataFile, null);
     for (int i = 0; i < 1000; i++) {
@@ -117,7 +123,7 @@ public class TestLogFile {
       ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
       FlumeEventPointer ptr = logFileWriter.put(bytes);
       final int offset = ptr.getOffset();
-      executorService.submit(new Runnable() {
+      completionService.submit(new Runnable() {
         @Override
         public void run() {
           try {
@@ -130,7 +136,11 @@ public class TestLogFile {
             }
           }
         }
-      });
+      }, null);
+    }
+
+    for(int i = 0; i < 1000; i++) {
+      completionService.take();
     }
     // first try and throw failures
     for(Throwable throwable : errors) {
@@ -142,7 +152,8 @@ public class TestLogFile {
     }
   }
   @Test
-  public void testReader() throws InterruptedException, IOException {
+  public void testReader() throws InterruptedException, IOException,
+    CorruptEventException {
     Map<Integer, Put> puts = Maps.newHashMap();
     for (int i = 0; i < 1000; i++) {
       FlumeEvent eventIn = TestUtils.newPersistableEvent();
@@ -169,7 +180,8 @@ public class TestLogFile {
   }
 
   @Test
-  public void testReaderOldMetaFile() throws InterruptedException, IOException {
+  public void testReaderOldMetaFile() throws InterruptedException,
+    IOException, CorruptEventException {
     Map<Integer, Put> puts = Maps.newHashMap();
     for (int i = 0; i < 1000; i++) {
       FlumeEvent eventIn = TestUtils.newPersistableEvent();
@@ -204,7 +216,8 @@ public class TestLogFile {
   }
 
     @Test
-  public void testReaderTempMetaFile() throws InterruptedException, IOException {
+  public void testReaderTempMetaFile() throws InterruptedException,
+      IOException, CorruptEventException {
     Map<Integer, Put> puts = Maps.newHashMap();
     for (int i = 0; i < 1000; i++) {
       FlumeEvent eventIn = TestUtils.newPersistableEvent();
@@ -260,4 +273,92 @@ public class TestLogFile {
     Assert.assertEquals(3, metaData.getCheckpointPosition());
     Assert.assertEquals(4, metaData.getCheckpointWriteOrderID());
   }
+
+  @Test (expected = CorruptEventException.class)
+  public void testPutGetCorruptEvent() throws Exception {
+    final LogFile.RandomReader logFileReader =
+      LogFileFactory.getRandomReader(dataFile, null);
+    final FlumeEvent eventIn = TestUtils.newPersistableEvent(2500);
+    final Put put = new Put(++transactionID, WriteOrderOracle.next(),
+      eventIn);
+    ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
+    FlumeEventPointer ptr = logFileWriter.put(bytes);
+    logFileWriter.commit(TransactionEventRecord.toByteBuffer(new Commit
+      (transactionID, WriteOrderOracle.next())));
+    final int offset = ptr.getOffset();
+    RandomAccessFile writer = new RandomAccessFile(dataFile, "rw");
+    writer.seek(offset + 1500);
+    writer.write((byte) 45);
+    writer.write((byte) 12);
+    writer.getFD().sync();
+    logFileReader.get(offset);
+
+    // Should have thrown an exception by now.
+    Assert.fail();
+
+  }
+
+  @Test (expected = NoopRecordException.class)
+  public void testPutGetNoopEvent() throws Exception {
+    final LogFile.RandomReader logFileReader =
+      LogFileFactory.getRandomReader(dataFile, null);
+    final FlumeEvent eventIn = TestUtils.newPersistableEvent(2500);
+    final Put put = new Put(++transactionID, WriteOrderOracle.next(),
+      eventIn);
+    ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
+    FlumeEventPointer ptr = logFileWriter.put(bytes);
+    logFileWriter.commit(TransactionEventRecord.toByteBuffer(new Commit
+      (transactionID, WriteOrderOracle.next())));
+    final int offset = ptr.getOffset();
+    LogFile.OperationRecordUpdater updater = new LogFile
+      .OperationRecordUpdater(dataFile);
+    updater.markRecordAsNoop(offset);
+    logFileReader.get(offset);
+
+    // Should have thrown an exception by now.
+    Assert.fail();
+  }
+
+  @Test
+  public void testOperationRecordUpdater() throws Exception {
+    File tempDir = Files.createTempDir();
+    File temp = new File(tempDir, "temp");
+    final RandomAccessFile tempFile = new RandomAccessFile(temp, "rw");
+    for(int i = 0; i < 5000; i++) {
+      tempFile.write(LogFile.OP_RECORD);
+    }
+    tempFile.seek(0);
+    LogFile.OperationRecordUpdater recordUpdater = new LogFile
+      .OperationRecordUpdater(temp);
+    //Convert every 10th byte into a noop byte
+    for(int i = 0; i < 5000; i+=10) {
+      recordUpdater.markRecordAsNoop(i);
+    }
+    recordUpdater.close();
+
+    tempFile.seek(0);
+    // Verify every 10th byte is actually a NOOP
+    for(int i = 0; i < 5000; i+=10) {
+      tempFile.seek(i);
+      Assert.assertEquals(LogFile.OP_NOOP, tempFile.readByte());
+    }
+
+  }
+
+  @Test
+  public void testOpRecordUpdaterWithFlumeEvents() throws Exception{
+    final FlumeEvent eventIn = TestUtils.newPersistableEvent(2500);
+    final Put put = new Put(++transactionID, WriteOrderOracle.next(),
+      eventIn);
+    ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
+    FlumeEventPointer ptr = logFileWriter.put(bytes);
+    logFileWriter.commit(TransactionEventRecord.toByteBuffer(new Commit
+      (transactionID, WriteOrderOracle.next())));
+    final int offset = ptr.getOffset();
+    LogFile.OperationRecordUpdater updater = new LogFile
+      .OperationRecordUpdater(dataFile);
+    updater.markRecordAsNoop(offset);
+    RandomAccessFile fileReader = new RandomAccessFile(dataFile, "rw");
+    Assert.assertEquals(LogFile.OP_NOOP, fileReader.readByte());
+  }
 }

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
index f403422..eb0ce04 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
@@ -53,7 +53,7 @@ public class TestTransactionEventRecordV3 {
         commit.getRecordType());
   }
   @Test
-  public void testPutSerialization() throws IOException {
+  public void testPutSerialization() throws IOException, CorruptEventException {
     Map<String, String> headers = new HashMap<String, String>();
     headers.put("key", "value");
     Put in = new Put(System.currentTimeMillis(),
@@ -70,7 +70,8 @@ public class TestTransactionEventRecordV3 {
     Assert.assertTrue(Arrays.equals(in.getEvent().getBody(), out.getEvent().getBody()));
   }
   @Test
-  public void testPutSerializationNullHeader() throws IOException {
+  public void testPutSerializationNullHeader() throws IOException,
+    CorruptEventException {
     Put in = new Put(System.currentTimeMillis(),
         WriteOrderOracle.next(),
         new FlumeEvent(null, new byte[0]));
@@ -84,7 +85,8 @@ public class TestTransactionEventRecordV3 {
     Assert.assertTrue(Arrays.equals(in.getEvent().getBody(), out.getEvent().getBody()));
   }
   @Test
-  public void testTakeSerialization() throws IOException {
+  public void testTakeSerialization() throws IOException,
+    CorruptEventException {
     Take in = new Take(System.currentTimeMillis(),
         WriteOrderOracle.next(), 10, 20);
     Take out = (Take)TransactionEventRecord.fromByteArray(toByteArray(in));
@@ -97,7 +99,8 @@ public class TestTransactionEventRecordV3 {
   }
 
   @Test
-  public void testRollbackSerialization() throws IOException {
+  public void testRollbackSerialization() throws IOException,
+    CorruptEventException {
     Rollback in = new Rollback(System.currentTimeMillis(),
         WriteOrderOracle.next());
     Rollback out = (Rollback)TransactionEventRecord.fromByteArray(toByteArray(in));
@@ -108,7 +111,8 @@ public class TestTransactionEventRecordV3 {
   }
 
   @Test
-  public void testCommitSerialization() throws IOException {
+  public void testCommitSerialization() throws IOException,
+    CorruptEventException {
     Commit in = new Commit(System.currentTimeMillis(),
         WriteOrderOracle.next());
     Commit out = (Commit)TransactionEventRecord.fromByteArray(toByteArray(in));
@@ -119,7 +123,7 @@ public class TestTransactionEventRecordV3 {
   }
 
   @Test
-  public void testBadType() throws IOException {
+  public void testBadType() throws IOException, CorruptEventException {
     TransactionEventRecord in = mock(TransactionEventRecord.class);
     when(in.getRecordType()).thenReturn(Short.MIN_VALUE);
     try {

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
index 563dbcc..a5ab45a 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
@@ -29,6 +29,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URL;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -64,6 +65,16 @@ public class TestUtils {
     return event;
   }
 
+  public static FlumeEvent newPersistableEvent(int size) {
+    Map<String, String> headers = Maps.newHashMap();
+    String timestamp = String.valueOf(System.currentTimeMillis());
+    headers.put("timestamp", timestamp);
+    byte[] data = new byte[size];
+    Arrays.fill(data, (byte) 54);
+    FlumeEvent event = new FlumeEvent(headers, data);
+    return event;
+  }
+
   public static DataInput toDataInput(Writable writable) throws IOException {
     ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
     DataOutputStream dataOutput = new DataOutputStream(byteOutput);
@@ -129,25 +140,47 @@ public class TestUtils {
             .invoke(true));
   }
 
+  public static Set<String> takeEvents(Channel channel, int batchSize)
+    throws Exception {
+    return takeEvents(channel, batchSize, false);
+  }
+
+  public static Set<String> takeEvents(Channel channel,
+          int batchSize, boolean checkForCorruption) throws Exception {
+    return takeEvents(channel, batchSize, Integer.MAX_VALUE, checkForCorruption);
+  }
+
   public static Set<String> takeEvents(Channel channel,
-          int batchSize) throws Exception {
-    return takeEvents(channel, batchSize, Integer.MAX_VALUE);
+    int batchSize, int numEvents) throws Exception {
+    return takeEvents(channel, batchSize, numEvents, false);
   }
 
   public static Set<String> takeEvents(Channel channel,
-          int batchSize, int numEvents) throws Exception {
+          int batchSize, int numEvents, boolean checkForCorruption) throws
+    Exception {
     Set<String> result = Sets.newHashSet();
     for (int i = 0; i < numEvents; i += batchSize) {
       Transaction transaction = channel.getTransaction();
       try {
         transaction.begin();
         for (int j = 0; j < batchSize; j++) {
-          Event event = null;
+          Event event;
           try {
             event = channel.take();
           } catch (ChannelException ex) {
-            Assert.assertTrue(ex.getMessage().startsWith(
-                "Take list for FileBackedTransaction, capacity"));
+            Throwable th = ex;
+            String msg;
+            if(checkForCorruption) {
+              msg = "Corrupt event found. Please run File Channel";
+              th = ex.getCause();
+            } else {
+              msg = "Take list for FileBackedTransaction, capacity";
+            }
+            Assert.assertTrue(th.getMessage().startsWith(
+                msg));
+            if(checkForCorruption) {
+              throw (Exception) th;
+            }
             transaction.commit();
             return result;
           }
@@ -168,15 +201,19 @@ public class TestUtils {
     }
     return result;
   }
-  public static Set<String> consumeChannel(Channel channel)
-      throws Exception {
+
+  public static Set<String> consumeChannel(Channel channel) throws Exception {
+    return consumeChannel(channel, false);
+  }
+  public static Set<String> consumeChannel(Channel channel,
+    boolean checkForCorruption) throws Exception {
     Set<String> result = Sets.newHashSet();
     int[] batchSizes = new int[] {
         1000, 100, 10, 1
     };
     for (int i = 0; i < batchSizes.length; i++) {
       while(true) {
-        Set<String> batch = takeEvents(channel, batchSizes[i]);
+        Set<String> batch = takeEvents(channel, batchSizes[i], checkForCorruption);
         if(batch.isEmpty()) {
           break;
         }

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml
index e266272..9a5f64e 100644
--- a/flume-ng-dist/pom.xml
+++ b/flume-ng-dist/pom.xml
@@ -137,6 +137,10 @@
       <artifactId>flume-ng-log4jappender</artifactId>
       <classifier>jar-with-dependencies</classifier>
     </dependency>
+    <dependency>
+      <groupId>org.apache.flume</groupId>
+      <artifactId>flume-tools</artifactId>
+    </dependency>
   </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-dist/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/flume-ng-dist/src/main/assembly/bin.xml b/flume-ng-dist/src/main/assembly/bin.xml
index 74f0608..925a54a 100644
--- a/flume-ng-dist/src/main/assembly/bin.xml
+++ b/flume-ng-dist/src/main/assembly/bin.xml
@@ -95,6 +95,7 @@
         <exclude>flume-ng-legacy-sources/**</exclude>
         <exclude>flume-ng-clients/**</exclude>
         <exclude>flume-ng-embedded-agent/**</exclude>
+        <exclude>flume-tools/**</exclude>
         <exclude>**/target/**</exclude>
         <exclude>**/.classpath</exclude>
         <exclude>**/.project</exclude>

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-dist/src/main/assembly/src.xml
----------------------------------------------------------------------
diff --git a/flume-ng-dist/src/main/assembly/src.xml b/flume-ng-dist/src/main/assembly/src.xml
index 7fafab8..5b86994 100644
--- a/flume-ng-dist/src/main/assembly/src.xml
+++ b/flume-ng-dist/src/main/assembly/src.xml
@@ -47,6 +47,7 @@
         <include>org.apache.flume:flume-ng-legacy-sources</include>
         <include>org.apache.flume:flume-ng-clients</include>
         <include>org.apache.flume:flume-ng-embedded-agent</include>
+        <include>org.apache.flume:flume-tools</include>
       </includes>
 
       <sources>
@@ -89,6 +90,7 @@
         <exclude>flume-ng-legacy-sources/**</exclude>
         <exclude>flume-ng-clients/**</exclude>
         <exclude>flume-ng-embedded-agent/**</exclude>
+        <exclude>flume-tools/**</exclude>
         <exclude>**/target/**</exclude>
         <exclude>**/.classpath</exclude>
         <exclude>**/.project</exclude>

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/pom.xml
----------------------------------------------------------------------
diff --git a/flume-tools/pom.xml b/flume-tools/pom.xml
new file mode 100644
index 0000000..386c766
--- /dev/null
+++ b/flume-tools/pom.xml
@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <artifactId>flume-parent</artifactId>
+    <groupId>org.apache.flume</groupId>
+    <version>1.4.0-SNAPSHOT</version>
+  </parent>
+
+  <groupId>org.apache.flume</groupId>
+  <artifactId>flume-tools</artifactId>
+
+  <name>Flume NG Tools</name>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.flume.flume-ng-channels</groupId>
+      <artifactId>flume-file-channel</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+
+
+    <dependency>
+      <groupId>commons-cli</groupId>
+      <artifactId>commons-cli</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.easytesting</groupId>
+      <artifactId>fest-reflect</artifactId>
+      <version>1.4</version>
+    </dependency>
+
+  </dependencies>
+  <profiles>
+
+    <profile>
+      <id>hadoop-1.0</id>
+      <activation>
+        <property>
+          <name>!hadoop.profile</name>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>1.0.1</hadoop.version>
+        <hadoop.common.artifact.id>hadoop-core</hadoop.common.artifact.id>
+      </properties>
+      <dependencies>
+
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-test</artifactId>
+          <scope>test</scope>
+        </dependency>
+
+        <!-- required because the hadoop-core pom is missing these deps
+            and MiniDFSCluster pulls in the webhdfs classes -->
+        <dependency>
+          <groupId>com.sun.jersey</groupId>
+          <artifactId>jersey-core</artifactId>
+          <scope>test</scope>
+        </dependency>
+
+      </dependencies>
+    </profile>
+
+    <profile>
+      <id>hadoop-2</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>2</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>2.0.0-alpha</hadoop.version>
+        <hadoop.common.artifact.id>hadoop-common</hadoop.common.artifact.id>
+      </properties>
+      <dependencies>
+
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs</artifactId>
+          <optional>true</optional>
+        </dependency>
+
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-auth</artifactId>
+          <optional>true</optional>
+        </dependency>
+
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <scope>test</scope>
+        </dependency>
+
+      </dependencies>
+    </profile>
+
+  </profiles>
+
+</project>

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java b/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
new file mode 100644
index 0000000..aa24fa5
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.channel.file.CorruptEventException;
+import org.apache.flume.channel.file.Log;
+import org.apache.flume.channel.file.LogFile;
+import org.apache.flume.channel.file.LogFileV3;
+import org.apache.flume.channel.file.LogRecord;
+import org.apache.flume.channel.file.Serialization;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class FileChannelIntegrityTool implements FlumeTool {
+  public static final Logger LOG = LoggerFactory.getLogger
+    (FileChannelIntegrityTool.class);
+
+  private final List<File> dataDirs = new ArrayList<File>();
+
+  @Override
+  public void run(String[] args) throws IOException, ParseException {
+    boolean shouldContinue = parseCommandLineOpts(args);
+    if(!shouldContinue) {
+      LOG.error("Could not parse command line options. Exiting ...");
+      System.exit(1);
+    }
+    for(File dataDir : dataDirs) {
+      File[] dataFiles = dataDir.listFiles(new FilenameFilter() {
+        @Override
+        public boolean accept(File dir, String name) {
+          if(!name.endsWith(Serialization.METADATA_FILENAME)
+            && !name.endsWith(Serialization.METADATA_TMP_FILENAME)
+            && !name.endsWith(Serialization.OLD_METADATA_FILENAME)
+            && !name.equals(Log.FILE_LOCK)) {
+            return true;
+          }
+          return false;
+        }
+      });
+      if (dataFiles != null && dataFiles.length > 0) {
+        for (File dataFile : dataFiles) {
+          LOG.info("Checking for corruption in " + dataFile.toString());
+          LogFile.SequentialReader reader =
+            new LogFileV3.SequentialReader(dataFile, null);
+          LogFile.OperationRecordUpdater updater = new LogFile
+            .OperationRecordUpdater(dataFile);
+          boolean fileDone = false;
+          boolean fileBackedup = false;
+          while (!fileDone) {
+            long eventPosition = 0;
+            try {
+              // This depends on the fact that events are of the form:
+              // Type, length, data.
+              eventPosition = reader.getPosition();
+              // Try to get the record, if the checksums don't match,
+              // this will throw a CorruptEventException - so the real logic
+              // is in the catch block below.
+              LogRecord record = reader.next();
+              if (record != null) {
+                record.getEvent();
+              } else {
+                fileDone = true;
+              }
+            } catch (CorruptEventException e) {
+              LOG.warn("Corruption found in " + dataFile.toString() + " at "
+                + eventPosition);
+              if (!fileBackedup) {
+                Serialization.copyFile(dataFile, new File(dataFile.getParent(),
+                  dataFile.getName() + ".bak"));
+                fileBackedup = true;
+              }
+              updater.markRecordAsNoop(eventPosition);
+            }
+          }
+          updater.close();
+          reader.close();
+        }
+      }
+    }
+  }
+
+  private boolean parseCommandLineOpts(String[] args) throws ParseException {
+    Options options = new Options();
+    options
+      .addOption("l", "dataDirs", true, "Comma-separated list of data " +
+        "directories which the tool must verify. This option is mandatory")
+      .addOption("h", "help", false, "Display help");
+
+    CommandLineParser parser = new GnuParser();
+    CommandLine commandLine = parser.parse(options, args);
+    if(commandLine.hasOption("help")) {
+      new HelpFormatter().printHelp("java -jar fcintegritytool ",
+        options, true);
+      return false;
+    }
+    if(!commandLine.hasOption("dataDirs")) {
+      new HelpFormatter().printHelp("java -jar fcintegritytool ", "",
+        options, "dataDirs is required.", true);
+      return false;
+    } else {
+      String dataDirStr[] = commandLine.getOptionValue("dataDirs").split(",");
+      for(String dataDir : dataDirStr) {
+        File f = new File(dataDir);
+        if(!f.exists()) {
+          throw new FlumeException("Data directory, " + dataDir + " does not " +
+            "exist.");
+        }
+        dataDirs.add(f);
+      }
+    }
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java b/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
new file mode 100644
index 0000000..28da8fe
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+public interface FlumeTool {
+
+  public void run(String[] args) throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
new file mode 100644
index 0000000..f886c89
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+public enum FlumeToolType {
+  FCINTEGRITYTOOL(FileChannelIntegrityTool.class);
+
+  private final Class<? extends FlumeTool> klass;
+  private FlumeToolType(Class<? extends FlumeTool> klass) {
+    this.klass = klass;
+  }
+
+  public Class<? extends FlumeTool> getClassInstance() {
+    return this.klass;
+  }
+
+  public static String getNames() {
+    StringBuilder builder = new StringBuilder();
+    for(FlumeToolType type: values()) {
+      builder.append(type.name().toLowerCase() + "\n");
+    }
+    return builder.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
new file mode 100644
index 0000000..799ce85
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Arrays;
+
+public class FlumeToolsMain implements FlumeTool {
+
+  // Does the actual work in the run method so we can test it without actually
+  // having to start another process.
+
+  public static void main(String[] args) throws Exception {
+    new FlumeToolsMain().run(args);
+  }
+
+  private FlumeToolsMain() {
+    //No op.
+  }
+
+  @Override
+  public void run(String[] args) throws Exception{
+    String error = "Expected name of tool and arguments for" +
+      " tool to be passed in on the command line. Please pass one of the " +
+      "following as arguments to this command: \n";
+    StringBuilder builder = new StringBuilder(error);
+    for(FlumeToolType type : FlumeToolType.values()) {
+      builder.append(type.name()).append("\n");
+    }
+    if(args == null || args.length == 0) {
+      System.out.println(builder.toString());
+      System.exit(1);
+    }
+    String toolName = args[0];
+    FlumeTool tool = null;
+    for(FlumeToolType type : FlumeToolType.values()) {
+      if(toolName.equalsIgnoreCase(type.name())) {
+        tool = type.getClassInstance().newInstance();
+        break;
+      }
+    }
+    Preconditions.checkNotNull(tool, "Cannot find tool matching " + toolName
+      + ". Please select one of: \n " + FlumeToolType.getNames());
+    if (args.length == 1) {
+      tool.run(new String[0]);
+    } else {
+      tool.run(Arrays.asList(args).subList(1, args.length).
+        toArray(new String[0]));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java b/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
new file mode 100644
index 0000000..d328671
--- /dev/null
+++ b/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.Transaction;
+import org.apache.flume.channel.file.FileChannel;
+import org.apache.flume.channel.file.FileChannelConfiguration;
+import org.apache.flume.channel.file.Log;
+import org.apache.flume.channel.file.LogFile;
+import org.apache.flume.channel.file.LogFileV3;
+import org.apache.flume.channel.file.LogRecord;
+import org.apache.flume.channel.file.Serialization;
+import org.apache.flume.channel.file.WriteOrderOracle;
+import org.apache.flume.event.EventBuilder;
+import org.junit.After;
+import org.junit.AfterClass;
+import static org.fest.reflect.core.Reflection.*;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.RandomAccessFile;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+
+public class TestFileChannelIntegrityTool {
+  private static File baseDir;
+  private static File origCheckpointDir;
+  private static File origDataDir;
+  private static Event event;
+  private static Context ctx;
+
+  private File checkpointDir;
+  private File dataDir;
+
+
+  @BeforeClass
+  public static void setUpClass() throws Exception{
+    createDataFiles();
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    checkpointDir = new File(baseDir, "checkpoint");
+    dataDir = new File(baseDir, "dataDir");
+    Assert.assertTrue(checkpointDir.mkdirs() || checkpointDir.isDirectory());
+    Assert.assertTrue(dataDir.mkdirs() || dataDir.isDirectory());
+    File[] dataFiles = origDataDir.listFiles(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        if(name.contains("lock")) {
+          return false;
+        }
+        return true;
+      }
+    });
+    for(File dataFile : dataFiles) {
+      Serialization.copyFile(dataFile, new File(dataDir, dataFile.getName()));
+    }
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    FileUtils.deleteDirectory(checkpointDir);
+    FileUtils.deleteDirectory(dataDir);
+  }
+
+  @AfterClass
+  public static void tearDownClass() throws Exception {
+    FileUtils.deleteDirectory(origCheckpointDir);
+    FileUtils.deleteDirectory(origDataDir);
+  }
+
+  @Test
+  public void testFixCorruptRecords() throws Exception {
+    doTestFixCorruptEvents(false);
+  }
+  @Test
+  public void testFixCorruptRecordsWithCheckpoint() throws Exception {
+    doTestFixCorruptEvents(true);
+  }
+
+  public void doTestFixCorruptEvents(boolean withCheckpoint) throws Exception {
+    Set<String> corruptFiles = new HashSet<String>();
+    File[] files = dataDir.listFiles(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        if(name.contains("lock") || name.contains("meta")) {
+          return false;
+        }
+        return true;
+      }
+    });
+    Random random = new Random();
+    int corrupted = 0;
+    for (File dataFile : files) {
+      LogFile.SequentialReader reader =
+        new LogFileV3.SequentialReader(dataFile, null);
+      RandomAccessFile handle = new RandomAccessFile(dataFile, "rw");
+      long eventPosition1 = reader.getPosition();
+      LogRecord rec = reader.next();
+      //No point corrupting commits, so ignore them
+      if(rec == null ||
+        rec.getEvent().getClass().getName().
+          equals("org.apache.flume.channel.file.Commit")) {
+        handle.close();
+        reader.close();
+        continue;
+      }
+      long eventPosition2 = reader.getPosition();
+      rec = reader.next();
+      handle.seek(eventPosition1 + 100);
+      handle.writeInt(random.nextInt());
+      corrupted++;
+      corruptFiles.add(dataFile.getName());
+      if (rec == null ||
+        rec.getEvent().getClass().getName().
+          equals("org.apache.flume.channel.file.Commit")) {
+        handle.close();
+        reader.close();
+        continue;
+      }
+      handle.seek(eventPosition2 + 100);
+      handle.writeInt(random.nextInt());
+      corrupted++;
+      handle.close();
+      reader.close();
+
+    }
+    FileChannelIntegrityTool tool = new FileChannelIntegrityTool();
+    tool.run(new String[] {"-l", dataDir.toString()});
+    FileChannel channel = new FileChannel();
+    channel.setName("channel");
+    String cp;
+    if(withCheckpoint) {
+      cp = origCheckpointDir.toString();
+    } else {
+      FileUtils.deleteDirectory(checkpointDir);
+      Assert.assertTrue(checkpointDir.mkdirs());
+      cp = checkpointDir.toString();
+    }
+    ctx.put(FileChannelConfiguration.CHECKPOINT_DIR,cp);
+    ctx.put(FileChannelConfiguration.DATA_DIRS, dataDir.toString());
+    channel.configure(ctx);
+    channel.start();
+    Transaction tx = channel.getTransaction();
+    tx.begin();
+    int i = 0;
+    while(channel.take() != null) {
+      i++;
+    }
+    tx.commit();
+    tx.close();
+    channel.stop();
+    Assert.assertEquals(25 - corrupted, i);
+    files = dataDir.listFiles(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        if(name.contains(".bak")) {
+          return true;
+        }
+        return false;
+      }
+    });
+    Assert.assertEquals(corruptFiles.size(), files.length);
+    for(File file : files) {
+      String name = file.getName();
+      name = name.replaceAll(".bak", "");
+      Assert.assertTrue(corruptFiles.remove(name));
+    }
+    Assert.assertTrue(corruptFiles.isEmpty());
+  }
+
+  private static void createDataFiles() throws Exception {
+    final byte[] eventData = new byte[2000];
+    for(int i = 0; i < 2000; i++) {
+      eventData[i] = 1;
+    }
+    WriteOrderOracle.setSeed(System.currentTimeMillis());
+    event = EventBuilder.withBody(eventData);
+    baseDir = Files.createTempDir();
+    if(baseDir.exists()) {
+      FileUtils.deleteDirectory(baseDir);
+    }
+    baseDir = Files.createTempDir();
+    origCheckpointDir = new File(baseDir, "chkpt");
+    Assert.assertTrue(origCheckpointDir.mkdirs() || origCheckpointDir.isDirectory());
+    origDataDir = new File(baseDir, "data");
+    Assert.assertTrue(origDataDir.mkdirs() || origDataDir.isDirectory());
+    FileChannel channel = new FileChannel();
+    channel.setName("channel");
+    ctx = new Context();
+    ctx.put(FileChannelConfiguration.CAPACITY, "1000");
+    ctx.put(FileChannelConfiguration.CHECKPOINT_DIR, origCheckpointDir.toString());
+    ctx.put(FileChannelConfiguration.DATA_DIRS, origDataDir.toString());
+    ctx.put(FileChannelConfiguration.MAX_FILE_SIZE, "10000");
+    ctx.put(FileChannelConfiguration.TRANSACTION_CAPACITY, "100");
+    channel.configure(ctx);
+    channel.start();
+    for (int j = 0; j < 5; j++) {
+      Transaction tx = channel.getTransaction();
+      tx.begin();
+      for (int i = 0; i < 5; i++) {
+        channel.put(event);
+      }
+      tx.commit();
+      tx.close();
+    }
+    Log log = field("log")
+      .ofType(Log.class)
+      .in(channel)
+      .get();
+
+    Assert.assertTrue("writeCheckpoint returned false",
+      method("writeCheckpoint")
+        .withReturnType(Boolean.class)
+        .withParameterTypes(Boolean.class)
+        .in(log)
+        .invoke(true));
+    channel.stop();
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9982f5d..8026936 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,6 +63,7 @@ limitations under the License.
     <module>flume-ng-clients</module>
     <module>flume-ng-sdk</module>
     <module>flume-ng-tests</module>
+    <module>flume-tools</module>
   </modules>
 
   <profiles>
@@ -888,6 +889,12 @@ limitations under the License.
 
       <dependency>
         <groupId>org.apache.flume</groupId>
+        <artifactId>flume-tools</artifactId>
+        <version>1.4.0-SNAPSHOT</version>
+      </dependency>
+
+      <dependency>
+        <groupId>org.apache.flume</groupId>
         <artifactId>flume-ng-node</artifactId>
         <version>1.4.0-SNAPSHOT</version>
       </dependency>


[2/3] FLUME-1586: File Channel should support verifying integrity of individual events.

Posted by br...@apache.org.
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
index 4860ac2..6885832 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
@@ -28,29 +28,29 @@ public final class ProtosFactory {
   }
   public interface CheckpointOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required sfixed32 version = 1;
     boolean hasVersion();
     int getVersion();
-
+    
     // required sfixed64 writeOrderID = 2;
     boolean hasWriteOrderID();
     long getWriteOrderID();
-
+    
     // required sfixed32 queueSize = 3;
     boolean hasQueueSize();
     int getQueueSize();
-
+    
     // required sfixed32 queueHead = 4;
     boolean hasQueueHead();
     int getQueueHead();
-
+    
     // repeated .ActiveLog activeLogs = 5;
-    java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog>
+    java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> 
         getActiveLogsList();
     org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog getActiveLogs(int index);
     int getActiveLogsCount();
-    java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+    java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder> 
         getActiveLogsOrBuilderList();
     org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder getActiveLogsOrBuilder(
         int index);
@@ -63,26 +63,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private Checkpoint(boolean noInit) {}
-
+    
     private static final Checkpoint defaultInstance;
     public static Checkpoint getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public Checkpoint getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required sfixed32 version = 1;
     public static final int VERSION_FIELD_NUMBER = 1;
@@ -93,7 +93,7 @@ public final class ProtosFactory {
     public int getVersion() {
       return version_;
     }
-
+    
     // required sfixed64 writeOrderID = 2;
     public static final int WRITEORDERID_FIELD_NUMBER = 2;
     private long writeOrderID_;
@@ -103,7 +103,7 @@ public final class ProtosFactory {
     public long getWriteOrderID() {
       return writeOrderID_;
     }
-
+    
     // required sfixed32 queueSize = 3;
     public static final int QUEUESIZE_FIELD_NUMBER = 3;
     private int queueSize_;
@@ -113,7 +113,7 @@ public final class ProtosFactory {
     public int getQueueSize() {
       return queueSize_;
     }
-
+    
     // required sfixed32 queueHead = 4;
     public static final int QUEUEHEAD_FIELD_NUMBER = 4;
     private int queueHead_;
@@ -123,14 +123,14 @@ public final class ProtosFactory {
     public int getQueueHead() {
       return queueHead_;
     }
-
+    
     // repeated .ActiveLog activeLogs = 5;
     public static final int ACTIVELOGS_FIELD_NUMBER = 5;
     private java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> activeLogs_;
     public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> getActiveLogsList() {
       return activeLogs_;
     }
-    public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+    public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder> 
         getActiveLogsOrBuilderList() {
       return activeLogs_;
     }
@@ -144,7 +144,7 @@ public final class ProtosFactory {
         int index) {
       return activeLogs_.get(index);
     }
-
+    
     private void initFields() {
       version_ = 0;
       writeOrderID_ = 0L;
@@ -156,7 +156,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasVersion()) {
         memoizedIsInitialized = 0;
         return false;
@@ -182,7 +182,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -203,12 +203,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -234,14 +234,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -308,14 +308,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -329,17 +329,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -352,7 +352,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         version_ = 0;
@@ -371,20 +371,20 @@ public final class ProtosFactory {
         }
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint build() {
         org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint result = buildPartial();
         if (!result.isInitialized()) {
@@ -392,7 +392,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint result = buildPartial();
@@ -402,7 +402,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint result = new org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint(this);
         int from_bitField0_ = bitField0_;
@@ -436,7 +436,7 @@ public final class ProtosFactory {
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint)other);
@@ -445,7 +445,7 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.getDefaultInstance()) return this;
         if (other.hasVersion()) {
@@ -478,7 +478,7 @@ public final class ProtosFactory {
               activeLogsBuilder_ = null;
               activeLogs_ = other.activeLogs_;
               bitField0_ = (bitField0_ & ~0x00000010);
-              activeLogsBuilder_ =
+              activeLogsBuilder_ = 
                 com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
                    getActiveLogsFieldBuilder() : null;
             } else {
@@ -489,33 +489,33 @@ public final class ProtosFactory {
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasVersion()) {
-
+          
           return false;
         }
         if (!hasWriteOrderID()) {
-
+          
           return false;
         }
         if (!hasQueueSize()) {
-
+          
           return false;
         }
         if (!hasQueueHead()) {
-
+          
           return false;
         }
         for (int i = 0; i < getActiveLogsCount(); i++) {
           if (!getActiveLogs(i).isInitialized()) {
-
+            
             return false;
           }
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -568,9 +568,9 @@ public final class ProtosFactory {
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required sfixed32 version = 1;
       private int version_ ;
       public boolean hasVersion() {
@@ -591,7 +591,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed64 writeOrderID = 2;
       private long writeOrderID_ ;
       public boolean hasWriteOrderID() {
@@ -612,7 +612,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed32 queueSize = 3;
       private int queueSize_ ;
       public boolean hasQueueSize() {
@@ -633,7 +633,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed32 queueHead = 4;
       private int queueHead_ ;
       public boolean hasQueueHead() {
@@ -654,7 +654,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // repeated .ActiveLog activeLogs = 5;
       private java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> activeLogs_ =
         java.util.Collections.emptyList();
@@ -664,10 +664,10 @@ public final class ProtosFactory {
           bitField0_ |= 0x00000010;
          }
       }
-
+      
       private com.google.protobuf.RepeatedFieldBuilder<
           org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder> activeLogsBuilder_;
-
+      
       public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> getActiveLogsList() {
         if (activeLogsBuilder_ == null) {
           return java.util.Collections.unmodifiableList(activeLogs_);
@@ -805,7 +805,7 @@ public final class ProtosFactory {
           return activeLogsBuilder_.getMessageOrBuilder(index);
         }
       }
-      public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+      public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder> 
            getActiveLogsOrBuilderList() {
         if (activeLogsBuilder_ != null) {
           return activeLogsBuilder_.getMessageOrBuilderList();
@@ -822,12 +822,12 @@ public final class ProtosFactory {
         return getActiveLogsFieldBuilder().addBuilder(
             index, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDefaultInstance());
       }
-      public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder>
+      public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder> 
            getActiveLogsBuilderList() {
         return getActiveLogsFieldBuilder().getBuilderList();
       }
       private com.google.protobuf.RepeatedFieldBuilder<
-          org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+          org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder> 
           getActiveLogsFieldBuilder() {
         if (activeLogsBuilder_ == null) {
           activeLogsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
@@ -840,25 +840,25 @@ public final class ProtosFactory {
         }
         return activeLogsBuilder_;
       }
-
+      
       // @@protoc_insertion_point(builder_scope:Checkpoint)
     }
-
+    
     static {
       defaultInstance = new Checkpoint(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:Checkpoint)
   }
-
+  
   public interface ActiveLogOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required sfixed32 logFileID = 1;
     boolean hasLogFileID();
     int getLogFileID();
-
+    
     // required sfixed32 count = 2;
     boolean hasCount();
     int getCount();
@@ -871,26 +871,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private ActiveLog(boolean noInit) {}
-
+    
     private static final ActiveLog defaultInstance;
     public static ActiveLog getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public ActiveLog getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required sfixed32 logFileID = 1;
     public static final int LOGFILEID_FIELD_NUMBER = 1;
@@ -901,7 +901,7 @@ public final class ProtosFactory {
     public int getLogFileID() {
       return logFileID_;
     }
-
+    
     // required sfixed32 count = 2;
     public static final int COUNT_FIELD_NUMBER = 2;
     private int count_;
@@ -911,7 +911,7 @@ public final class ProtosFactory {
     public int getCount() {
       return count_;
     }
-
+    
     private void initFields() {
       logFileID_ = 0;
       count_ = 0;
@@ -920,7 +920,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasLogFileID()) {
         memoizedIsInitialized = 0;
         return false;
@@ -932,7 +932,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -944,12 +944,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -963,14 +963,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -1037,14 +1037,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -1058,17 +1058,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -1080,7 +1080,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         logFileID_ = 0;
@@ -1089,20 +1089,20 @@ public final class ProtosFactory {
         bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog build() {
         org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog result = buildPartial();
         if (!result.isInitialized()) {
@@ -1110,7 +1110,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog result = buildPartial();
@@ -1120,7 +1120,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog result = new org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog(this);
         int from_bitField0_ = bitField0_;
@@ -1137,7 +1137,7 @@ public final class ProtosFactory {
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog)other);
@@ -1146,7 +1146,7 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDefaultInstance()) return this;
         if (other.hasLogFileID()) {
@@ -1158,19 +1158,19 @@ public final class ProtosFactory {
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasLogFileID()) {
-
+          
           return false;
         }
         if (!hasCount()) {
-
+          
           return false;
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -1207,9 +1207,9 @@ public final class ProtosFactory {
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required sfixed32 logFileID = 1;
       private int logFileID_ ;
       public boolean hasLogFileID() {
@@ -1230,7 +1230,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed32 count = 2;
       private int count_ ;
       public boolean hasCount() {
@@ -1251,46 +1251,46 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // @@protoc_insertion_point(builder_scope:ActiveLog)
     }
-
+    
     static {
       defaultInstance = new ActiveLog(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:ActiveLog)
   }
-
+  
   public interface LogFileMetaDataOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required sfixed32 version = 1;
     boolean hasVersion();
     int getVersion();
-
+    
     // required sfixed32 logFileID = 2;
     boolean hasLogFileID();
     int getLogFileID();
-
+    
     // required sfixed64 checkpointPosition = 3;
     boolean hasCheckpointPosition();
     long getCheckpointPosition();
-
+    
     // required sfixed64 checkpointWriteOrderID = 4;
     boolean hasCheckpointWriteOrderID();
     long getCheckpointWriteOrderID();
-
+    
     // optional .LogFileEncryption encryption = 5;
     boolean hasEncryption();
     org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption getEncryption();
     org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder getEncryptionOrBuilder();
-
+    
     // optional sfixed64 backupCheckpointPosition = 6;
     boolean hasBackupCheckpointPosition();
     long getBackupCheckpointPosition();
-
+    
     // optional sfixed64 backupCheckpointWriteOrderID = 7;
     boolean hasBackupCheckpointWriteOrderID();
     long getBackupCheckpointWriteOrderID();
@@ -1303,26 +1303,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private LogFileMetaData(boolean noInit) {}
-
+    
     private static final LogFileMetaData defaultInstance;
     public static LogFileMetaData getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public LogFileMetaData getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required sfixed32 version = 1;
     public static final int VERSION_FIELD_NUMBER = 1;
@@ -1333,7 +1333,7 @@ public final class ProtosFactory {
     public int getVersion() {
       return version_;
     }
-
+    
     // required sfixed32 logFileID = 2;
     public static final int LOGFILEID_FIELD_NUMBER = 2;
     private int logFileID_;
@@ -1343,7 +1343,7 @@ public final class ProtosFactory {
     public int getLogFileID() {
       return logFileID_;
     }
-
+    
     // required sfixed64 checkpointPosition = 3;
     public static final int CHECKPOINTPOSITION_FIELD_NUMBER = 3;
     private long checkpointPosition_;
@@ -1353,7 +1353,7 @@ public final class ProtosFactory {
     public long getCheckpointPosition() {
       return checkpointPosition_;
     }
-
+    
     // required sfixed64 checkpointWriteOrderID = 4;
     public static final int CHECKPOINTWRITEORDERID_FIELD_NUMBER = 4;
     private long checkpointWriteOrderID_;
@@ -1363,7 +1363,7 @@ public final class ProtosFactory {
     public long getCheckpointWriteOrderID() {
       return checkpointWriteOrderID_;
     }
-
+    
     // optional .LogFileEncryption encryption = 5;
     public static final int ENCRYPTION_FIELD_NUMBER = 5;
     private org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption encryption_;
@@ -1376,7 +1376,7 @@ public final class ProtosFactory {
     public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder getEncryptionOrBuilder() {
       return encryption_;
     }
-
+    
     // optional sfixed64 backupCheckpointPosition = 6;
     public static final int BACKUPCHECKPOINTPOSITION_FIELD_NUMBER = 6;
     private long backupCheckpointPosition_;
@@ -1386,7 +1386,7 @@ public final class ProtosFactory {
     public long getBackupCheckpointPosition() {
       return backupCheckpointPosition_;
     }
-
+    
     // optional sfixed64 backupCheckpointWriteOrderID = 7;
     public static final int BACKUPCHECKPOINTWRITEORDERID_FIELD_NUMBER = 7;
     private long backupCheckpointWriteOrderID_;
@@ -1396,7 +1396,7 @@ public final class ProtosFactory {
     public long getBackupCheckpointWriteOrderID() {
       return backupCheckpointWriteOrderID_;
     }
-
+    
     private void initFields() {
       version_ = 0;
       logFileID_ = 0;
@@ -1410,7 +1410,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasVersion()) {
         memoizedIsInitialized = 0;
         return false;
@@ -1436,7 +1436,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -1463,12 +1463,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -1502,14 +1502,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -1576,14 +1576,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -1597,17 +1597,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -1620,7 +1620,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         version_ = 0;
@@ -1643,20 +1643,20 @@ public final class ProtosFactory {
         bitField0_ = (bitField0_ & ~0x00000040);
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData build() {
         org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData result = buildPartial();
         if (!result.isInitialized()) {
@@ -1664,7 +1664,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData result = buildPartial();
@@ -1674,7 +1674,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData result = new org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData(this);
         int from_bitField0_ = bitField0_;
@@ -1715,7 +1715,7 @@ public final class ProtosFactory {
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData)other);
@@ -1724,7 +1724,7 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.getDefaultInstance()) return this;
         if (other.hasVersion()) {
@@ -1751,33 +1751,33 @@ public final class ProtosFactory {
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasVersion()) {
-
+          
           return false;
         }
         if (!hasLogFileID()) {
-
+          
           return false;
         }
         if (!hasCheckpointPosition()) {
-
+          
           return false;
         }
         if (!hasCheckpointWriteOrderID()) {
-
+          
           return false;
         }
         if (hasEncryption()) {
           if (!getEncryption().isInitialized()) {
-
+            
             return false;
           }
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -1843,9 +1843,9 @@ public final class ProtosFactory {
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required sfixed32 version = 1;
       private int version_ ;
       public boolean hasVersion() {
@@ -1866,7 +1866,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed32 logFileID = 2;
       private int logFileID_ ;
       public boolean hasLogFileID() {
@@ -1887,7 +1887,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed64 checkpointPosition = 3;
       private long checkpointPosition_ ;
       public boolean hasCheckpointPosition() {
@@ -1908,7 +1908,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed64 checkpointWriteOrderID = 4;
       private long checkpointWriteOrderID_ ;
       public boolean hasCheckpointWriteOrderID() {
@@ -1929,7 +1929,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // optional .LogFileEncryption encryption = 5;
       private org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption encryption_ = org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDefaultInstance();
       private com.google.protobuf.SingleFieldBuilder<
@@ -2007,7 +2007,7 @@ public final class ProtosFactory {
         }
       }
       private com.google.protobuf.SingleFieldBuilder<
-          org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.Builder, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder>
+          org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.Builder, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder> 
           getEncryptionFieldBuilder() {
         if (encryptionBuilder_ == null) {
           encryptionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
@@ -2019,7 +2019,7 @@ public final class ProtosFactory {
         }
         return encryptionBuilder_;
       }
-
+      
       // optional sfixed64 backupCheckpointPosition = 6;
       private long backupCheckpointPosition_ ;
       public boolean hasBackupCheckpointPosition() {
@@ -2040,7 +2040,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // optional sfixed64 backupCheckpointWriteOrderID = 7;
       private long backupCheckpointWriteOrderID_ ;
       public boolean hasBackupCheckpointWriteOrderID() {
@@ -2061,29 +2061,29 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // @@protoc_insertion_point(builder_scope:LogFileMetaData)
     }
-
+    
     static {
       defaultInstance = new LogFileMetaData(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:LogFileMetaData)
   }
-
+  
   public interface LogFileEncryptionOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required string cipherProvider = 1;
     boolean hasCipherProvider();
     String getCipherProvider();
-
+    
     // required string keyAlias = 2;
     boolean hasKeyAlias();
     String getKeyAlias();
-
+    
     // optional bytes parameters = 3;
     boolean hasParameters();
     com.google.protobuf.ByteString getParameters();
@@ -2096,26 +2096,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private LogFileEncryption(boolean noInit) {}
-
+    
     private static final LogFileEncryption defaultInstance;
     public static LogFileEncryption getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public LogFileEncryption getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required string cipherProvider = 1;
     public static final int CIPHERPROVIDER_FIELD_NUMBER = 1;
@@ -2128,7 +2128,7 @@ public final class ProtosFactory {
       if (ref instanceof String) {
         return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs =
+        com.google.protobuf.ByteString bs = 
             (com.google.protobuf.ByteString) ref;
         String s = bs.toStringUtf8();
         if (com.google.protobuf.Internal.isValidUtf8(bs)) {
@@ -2140,7 +2140,7 @@ public final class ProtosFactory {
     private com.google.protobuf.ByteString getCipherProviderBytes() {
       java.lang.Object ref = cipherProvider_;
       if (ref instanceof String) {
-        com.google.protobuf.ByteString b =
+        com.google.protobuf.ByteString b = 
             com.google.protobuf.ByteString.copyFromUtf8((String) ref);
         cipherProvider_ = b;
         return b;
@@ -2148,7 +2148,7 @@ public final class ProtosFactory {
         return (com.google.protobuf.ByteString) ref;
       }
     }
-
+    
     // required string keyAlias = 2;
     public static final int KEYALIAS_FIELD_NUMBER = 2;
     private java.lang.Object keyAlias_;
@@ -2160,7 +2160,7 @@ public final class ProtosFactory {
       if (ref instanceof String) {
         return (String) ref;
       } else {
-        com.google.protobuf.ByteString bs =
+        com.google.protobuf.ByteString bs = 
             (com.google.protobuf.ByteString) ref;
         String s = bs.toStringUtf8();
         if (com.google.protobuf.Internal.isValidUtf8(bs)) {
@@ -2172,7 +2172,7 @@ public final class ProtosFactory {
     private com.google.protobuf.ByteString getKeyAliasBytes() {
       java.lang.Object ref = keyAlias_;
       if (ref instanceof String) {
-        com.google.protobuf.ByteString b =
+        com.google.protobuf.ByteString b = 
             com.google.protobuf.ByteString.copyFromUtf8((String) ref);
         keyAlias_ = b;
         return b;
@@ -2180,7 +2180,7 @@ public final class ProtosFactory {
         return (com.google.protobuf.ByteString) ref;
       }
     }
-
+    
     // optional bytes parameters = 3;
     public static final int PARAMETERS_FIELD_NUMBER = 3;
     private com.google.protobuf.ByteString parameters_;
@@ -2190,7 +2190,7 @@ public final class ProtosFactory {
     public com.google.protobuf.ByteString getParameters() {
       return parameters_;
     }
-
+    
     private void initFields() {
       cipherProvider_ = "";
       keyAlias_ = "";
@@ -2200,7 +2200,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasCipherProvider()) {
         memoizedIsInitialized = 0;
         return false;
@@ -2212,7 +2212,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -2227,12 +2227,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -2250,14 +2250,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -2324,14 +2324,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -2345,17 +2345,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -2367,7 +2367,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         cipherProvider_ = "";
@@ -2378,20 +2378,20 @@ public final class ProtosFactory {
         bitField0_ = (bitField0_ & ~0x00000004);
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption build() {
         org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption result = buildPartial();
         if (!result.isInitialized()) {
@@ -2399,7 +2399,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption result = buildPartial();
@@ -2409,7 +2409,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption result = new org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption(this);
         int from_bitField0_ = bitField0_;
@@ -2430,7 +2430,7 @@ public final class ProtosFactory {
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption)other);
@@ -2439,7 +2439,7 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDefaultInstance()) return this;
         if (other.hasCipherProvider()) {
@@ -2454,19 +2454,19 @@ public final class ProtosFactory {
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasCipherProvider()) {
-
+          
           return false;
         }
         if (!hasKeyAlias()) {
-
+          
           return false;
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -2508,9 +2508,9 @@ public final class ProtosFactory {
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required string cipherProvider = 1;
       private java.lang.Object cipherProvider_ = "";
       public boolean hasCipherProvider() {
@@ -2546,7 +2546,7 @@ public final class ProtosFactory {
         cipherProvider_ = value;
         onChanged();
       }
-
+      
       // required string keyAlias = 2;
       private java.lang.Object keyAlias_ = "";
       public boolean hasKeyAlias() {
@@ -2582,7 +2582,7 @@ public final class ProtosFactory {
         keyAlias_ = value;
         onChanged();
       }
-
+      
       // optional bytes parameters = 3;
       private com.google.protobuf.ByteString parameters_ = com.google.protobuf.ByteString.EMPTY;
       public boolean hasParameters() {
@@ -2606,29 +2606,29 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // @@protoc_insertion_point(builder_scope:LogFileEncryption)
     }
-
+    
     static {
       defaultInstance = new LogFileEncryption(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:LogFileEncryption)
   }
-
+  
   public interface TransactionEventHeaderOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required sfixed32 type = 1;
     boolean hasType();
     int getType();
-
+    
     // required sfixed64 transactionID = 2;
     boolean hasTransactionID();
     long getTransactionID();
-
+    
     // required sfixed64 writeOrderID = 3;
     boolean hasWriteOrderID();
     long getWriteOrderID();
@@ -2641,26 +2641,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private TransactionEventHeader(boolean noInit) {}
-
+    
     private static final TransactionEventHeader defaultInstance;
     public static TransactionEventHeader getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public TransactionEventHeader getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required sfixed32 type = 1;
     public static final int TYPE_FIELD_NUMBER = 1;
@@ -2671,7 +2671,7 @@ public final class ProtosFactory {
     public int getType() {
       return type_;
     }
-
+    
     // required sfixed64 transactionID = 2;
     public static final int TRANSACTIONID_FIELD_NUMBER = 2;
     private long transactionID_;
@@ -2681,7 +2681,7 @@ public final class ProtosFactory {
     public long getTransactionID() {
       return transactionID_;
     }
-
+    
     // required sfixed64 writeOrderID = 3;
     public static final int WRITEORDERID_FIELD_NUMBER = 3;
     private long writeOrderID_;
@@ -2691,7 +2691,7 @@ public final class ProtosFactory {
     public long getWriteOrderID() {
       return writeOrderID_;
     }
-
+    
     private void initFields() {
       type_ = 0;
       transactionID_ = 0L;
@@ -2701,7 +2701,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasType()) {
         memoizedIsInitialized = 0;
         return false;
@@ -2717,7 +2717,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -2732,12 +2732,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -2755,14 +2755,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -2829,14 +2829,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -2850,17 +2850,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -2872,7 +2872,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         type_ = 0;
@@ -2883,20 +2883,20 @@ public final class ProtosFactory {
         bitField0_ = (bitField0_ & ~0x00000004);
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader build() {
         org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader result = buildPartial();
         if (!result.isInitialized()) {
@@ -2904,7 +2904,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader result = buildPartial();
@@ -2914,7 +2914,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader result = new org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader(this);
         int from_bitField0_ = bitField0_;
@@ -2935,7 +2935,7 @@ public final class ProtosFactory {
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader)other);
@@ -2944,7 +2944,7 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.getDefaultInstance()) return this;
         if (other.hasType()) {
@@ -2959,23 +2959,23 @@ public final class ProtosFactory {
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasType()) {
-
+          
           return false;
         }
         if (!hasTransactionID()) {
-
+          
           return false;
         }
         if (!hasWriteOrderID()) {
-
+          
           return false;
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3017,9 +3017,9 @@ public final class ProtosFactory {
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required sfixed32 type = 1;
       private int type_ ;
       public boolean hasType() {
@@ -3040,7 +3040,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed64 transactionID = 2;
       private long transactionID_ ;
       public boolean hasTransactionID() {
@@ -3061,7 +3061,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed64 writeOrderID = 3;
       private long writeOrderID_ ;
       public boolean hasWriteOrderID() {
@@ -3082,25 +3082,29 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // @@protoc_insertion_point(builder_scope:TransactionEventHeader)
     }
-
+    
     static {
       defaultInstance = new TransactionEventHeader(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:TransactionEventHeader)
   }
-
+  
   public interface PutOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required .FlumeEvent event = 1;
     boolean hasEvent();
     org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent getEvent();
     org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder getEventOrBuilder();
+    
+    // optional sfixed64 checksum = 2;
+    boolean hasChecksum();
+    long getChecksum();
   }
   public static final class Put extends
       com.google.protobuf.GeneratedMessage
@@ -3110,26 +3114,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private Put(boolean noInit) {}
-
+    
     private static final Put defaultInstance;
     public static Put getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public Put getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required .FlumeEvent event = 1;
     public static final int EVENT_FIELD_NUMBER = 1;
@@ -3143,15 +3147,26 @@ public final class ProtosFactory {
     public org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder getEventOrBuilder() {
       return event_;
     }
-
+    
+    // optional sfixed64 checksum = 2;
+    public static final int CHECKSUM_FIELD_NUMBER = 2;
+    private long checksum_;
+    public boolean hasChecksum() {
+      return ((bitField0_ & 0x00000002) == 0x00000002);
+    }
+    public long getChecksum() {
+      return checksum_;
+    }
+    
     private void initFields() {
       event_ = org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.getDefaultInstance();
+      checksum_ = 0L;
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasEvent()) {
         memoizedIsInitialized = 0;
         return false;
@@ -3163,38 +3178,45 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         output.writeMessage(1, event_);
       }
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        output.writeSFixed64(2, checksum_);
+      }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
           .computeMessageSize(1, event_);
       }
-      size += getUnknownFields().getSerializedSize();
+      if (((bitField0_ & 0x00000002) == 0x00000002)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeSFixed64Size(2, checksum_);
+      }
+      size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.Put parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -3261,14 +3283,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Put prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -3282,17 +3304,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.Put.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -3305,7 +3327,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         if (eventBuilder_ == null) {
@@ -3314,22 +3336,24 @@ public final class ProtosFactory {
           eventBuilder_.clear();
         }
         bitField0_ = (bitField0_ & ~0x00000001);
+        checksum_ = 0L;
+        bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Put.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Put getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Put.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Put build() {
         org.apache.flume.channel.file.proto.ProtosFactory.Put result = buildPartial();
         if (!result.isInitialized()) {
@@ -3337,7 +3361,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.Put buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.Put result = buildPartial();
@@ -3347,7 +3371,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Put buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.Put result = new org.apache.flume.channel.file.proto.ProtosFactory.Put(this);
         int from_bitField0_ = bitField0_;
@@ -3360,11 +3384,15 @@ public final class ProtosFactory {
         } else {
           result.event_ = eventBuilder_.build();
         }
+        if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+          to_bitField0_ |= 0x00000002;
+        }
+        result.checksum_ = checksum_;
         result.bitField0_ = to_bitField0_;
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Put) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Put)other);
@@ -3373,28 +3401,31 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Put other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.Put.getDefaultInstance()) return this;
         if (other.hasEvent()) {
           mergeEvent(other.getEvent());
         }
+        if (other.hasChecksum()) {
+          setChecksum(other.getChecksum());
+        }
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasEvent()) {
-
+          
           return false;
         }
         if (!getEvent().isInitialized()) {
-
+          
           return false;
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3427,12 +3458,17 @@ public final class ProtosFactory {
               setEvent(subBuilder.buildPartial());
               break;
             }
+            case 17: {
+              bitField0_ |= 0x00000002;
+              checksum_ = input.readSFixed64();
+              break;
+            }
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required .FlumeEvent event = 1;
       private org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent event_ = org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.getDefaultInstance();
       private com.google.protobuf.SingleFieldBuilder<
@@ -3510,7 +3546,7 @@ public final class ProtosFactory {
         }
       }
       private com.google.protobuf.SingleFieldBuilder<
-          org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.Builder, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder>
+          org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.Builder, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder> 
           getEventFieldBuilder() {
         if (eventBuilder_ == null) {
           eventBuilder_ = new com.google.protobuf.SingleFieldBuilder<
@@ -3522,25 +3558,46 @@ public final class ProtosFactory {
         }
         return eventBuilder_;
       }
-
+      
+      // optional sfixed64 checksum = 2;
+      private long checksum_ ;
+      public boolean hasChecksum() {
+        return ((bitField0_ & 0x00000002) == 0x00000002);
+      }
+      public long getChecksum() {
+        return checksum_;
+      }
+      public Builder setChecksum(long value) {
+        bitField0_ |= 0x00000002;
+        checksum_ = value;
+        onChanged();
+        return this;
+      }
+      public Builder clearChecksum() {
+        bitField0_ = (bitField0_ & ~0x00000002);
+        checksum_ = 0L;
+        onChanged();
+        return this;
+      }
+      
       // @@protoc_insertion_point(builder_scope:Put)
     }
-
+    
     static {
       defaultInstance = new Put(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:Put)
   }
-
+  
   public interface TakeOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required sfixed32 fileID = 1;
     boolean hasFileID();
     int getFileID();
-
+    
     // required sfixed32 offset = 2;
     boolean hasOffset();
     int getOffset();
@@ -3553,26 +3610,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private Take(boolean noInit) {}
-
+    
     private static final Take defaultInstance;
     public static Take getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public Take getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required sfixed32 fileID = 1;
     public static final int FILEID_FIELD_NUMBER = 1;
@@ -3583,7 +3640,7 @@ public final class ProtosFactory {
     public int getFileID() {
       return fileID_;
     }
-
+    
     // required sfixed32 offset = 2;
     public static final int OFFSET_FIELD_NUMBER = 2;
     private int offset_;
@@ -3593,7 +3650,7 @@ public final class ProtosFactory {
     public int getOffset() {
       return offset_;
     }
-
+    
     private void initFields() {
       fileID_ = 0;
       offset_ = 0;
@@ -3602,7 +3659,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasFileID()) {
         memoizedIsInitialized = 0;
         return false;
@@ -3614,7 +3671,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -3626,12 +3683,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -3645,14 +3702,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.Take parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -3719,14 +3776,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Take prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -3740,17 +3797,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.Take.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -3762,7 +3819,7 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         fileID_ = 0;
@@ -3771,20 +3828,20 @@ public final class ProtosFactory {
         bitField0_ = (bitField0_ & ~0x00000002);
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Take.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Take getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Take.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Take build() {
         org.apache.flume.channel.file.proto.ProtosFactory.Take result = buildPartial();
         if (!result.isInitialized()) {
@@ -3792,7 +3849,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.Take buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.Take result = buildPartial();
@@ -3802,7 +3859,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Take buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.Take result = new org.apache.flume.channel.file.proto.ProtosFactory.Take(this);
         int from_bitField0_ = bitField0_;
@@ -3819,7 +3876,7 @@ public final class ProtosFactory {
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Take) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Take)other);
@@ -3828,7 +3885,7 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Take other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.Take.getDefaultInstance()) return this;
         if (other.hasFileID()) {
@@ -3840,19 +3897,19 @@ public final class ProtosFactory {
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         if (!hasFileID()) {
-
+          
           return false;
         }
         if (!hasOffset()) {
-
+          
           return false;
         }
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3889,9 +3946,9 @@ public final class ProtosFactory {
           }
         }
       }
-
+      
       private int bitField0_;
-
+      
       // required sfixed32 fileID = 1;
       private int fileID_ ;
       public boolean hasFileID() {
@@ -3912,7 +3969,7 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // required sfixed32 offset = 2;
       private int offset_ ;
       public boolean hasOffset() {
@@ -3933,18 +3990,18 @@ public final class ProtosFactory {
         onChanged();
         return this;
       }
-
+      
       // @@protoc_insertion_point(builder_scope:Take)
     }
-
+    
     static {
       defaultInstance = new Take(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:Take)
   }
-
+  
   public interface RollbackOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
   }
@@ -3956,61 +4013,61 @@ public final class ProtosFactory {
       super(builder);
     }
     private Rollback(boolean noInit) {}
-
+    
     private static final Rollback defaultInstance;
     public static Rollback getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public Rollback getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_fieldAccessorTable;
     }
-
+    
     private void initFields() {
     }
     private byte memoizedIsInitialized = -1;
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       size += getUnknownFields().getSerializedSize();
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.Rollback parseFrom(
         com.google.protobuf.ByteString data)
         throws com.google.protobuf.InvalidProtocolBufferException {
@@ -4077,14 +4134,14 @@ public final class ProtosFactory {
       return newBuilder().mergeFrom(input, extensionRegistry)
                .buildParsed();
     }
-
+    
     public static Builder newBuilder() { return Builder.create(); }
     public Builder newBuilderForType() { return newBuilder(); }
     public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Rollback prototype) {
       return newBuilder().mergeFrom(prototype);
     }
     public Builder toBuilder() { return newBuilder(this); }
-
+    
     @java.lang.Override
     protected Builder newBuilderForType(
         com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -4098,17 +4155,17 @@ public final class ProtosFactory {
           getDescriptor() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_descriptor;
       }
-
+      
       protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
           internalGetFieldAccessorTable() {
         return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_fieldAccessorTable;
       }
-
+      
       // Construct using org.apache.flume.channel.file.proto.ProtosFactory.Rollback.newBuilder()
       private Builder() {
         maybeForceBuilderInitialization();
       }
-
+      
       private Builder(BuilderParent parent) {
         super(parent);
         maybeForceBuilderInitialization();
@@ -4120,25 +4177,25 @@ public final class ProtosFactory {
       private static Builder create() {
         return new Builder();
       }
-
+      
       public Builder clear() {
         super.clear();
         return this;
       }
-
+      
       public Builder clone() {
         return create().mergeFrom(buildPartial());
       }
-
+      
       public com.google.protobuf.Descriptors.Descriptor
           getDescriptorForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Rollback.getDescriptor();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Rollback getDefaultInstanceForType() {
         return org.apache.flume.channel.file.proto.ProtosFactory.Rollback.getDefaultInstance();
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Rollback build() {
         org.apache.flume.channel.file.proto.ProtosFactory.Rollback result = buildPartial();
         if (!result.isInitialized()) {
@@ -4146,7 +4203,7 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       private org.apache.flume.channel.file.proto.ProtosFactory.Rollback buildParsed()
           throws com.google.protobuf.InvalidProtocolBufferException {
         org.apache.flume.channel.file.proto.ProtosFactory.Rollback result = buildPartial();
@@ -4156,13 +4213,13 @@ public final class ProtosFactory {
         }
         return result;
       }
-
+      
       public org.apache.flume.channel.file.proto.ProtosFactory.Rollback buildPartial() {
         org.apache.flume.channel.file.proto.ProtosFactory.Rollback result = new org.apache.flume.channel.file.proto.ProtosFactory.Rollback(this);
         onBuilt();
         return result;
       }
-
+      
       public Builder mergeFrom(com.google.protobuf.Message other) {
         if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Rollback) {
           return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Rollback)other);
@@ -4171,17 +4228,17 @@ public final class ProtosFactory {
           return this;
         }
       }
-
+      
       public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Rollback other) {
         if (other == org.apache.flume.channel.file.proto.ProtosFactory.Rollback.getDefaultInstance()) return this;
         this.mergeUnknownFields(other.getUnknownFields());
         return this;
       }
-
+      
       public final boolean isInitialized() {
         return true;
       }
-
+      
       public Builder mergeFrom(
           com.google.protobuf.CodedInputStream input,
           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -4208,22 +4265,22 @@ public final class ProtosFactory {
           }
         }
       }
-
-
+      
+      
       // @@protoc_insertion_point(builder_scope:Rollback)
     }
-
+    
     static {
       defaultInstance = new Rollback(true);
       defaultInstance.initFields();
     }
-
+    
     // @@protoc_insertion_point(class_scope:Rollback)
   }
-
+  
   public interface CommitOrBuilder
       extends com.google.protobuf.MessageOrBuilder {
-
+    
     // required sfixed32 type = 1;
     boolean hasType();
     int getType();
@@ -4236,26 +4293,26 @@ public final class ProtosFactory {
       super(builder);
     }
     private Commit(boolean noInit) {}
-
+    
     private static final Commit defaultInstance;
     public static Commit getDefaultInstance() {
       return defaultInstance;
     }
-
+    
     public Commit getDefaultInstanceForType() {
       return defaultInstance;
     }
-
+    
     public static final com.google.protobuf.Descriptors.Descriptor
         getDescriptor() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Commit_descriptor;
     }
-
+    
     protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
         internalGetFieldAccessorTable() {
       return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Commit_fieldAccessorTable;
     }
-
+    
     private int bitField0_;
     // required sfixed32 type = 1;
     public static final int TYPE_FIELD_NUMBER = 1;
@@ -4266,7 +4323,7 @@ public final class ProtosFactory {
     public int getType() {
       return type_;
     }
-
+    
     private void initFields() {
       type_ = 0;
     }
@@ -4274,7 +4331,7 @@ public final class ProtosFactory {
     public final boolean isInitialized() {
       byte isInitialized = memoizedIsInitialized;
       if (isInitialized != -1) return isInitialized == 1;
-
+      
       if (!hasType()) {
         memoizedIsInitialized = 0;
         return false;
@@ -4282,7 +4339,7 @@ public final class ProtosFactory {
       memoizedIsInitialized = 1;
       return true;
     }
-
+    
     public void writeTo(com.google.protobuf.CodedOutputStream output)
                         throws java.io.IOException {
       getSerializedSize();
@@ -4291,12 +4348,12 @@ public final class ProtosFactory {
       }
       getUnknownFields().writeTo(output);
     }
-
+    
     private int memoizedSerializedSize = -1;
     public int getSerializedSize() {
       int size = memoizedSerializedSize;
       if (size != -1) return size;
-
+    
       size = 0;
       if (((bitField0_ & 0x00000001) == 0x00000001)) {
         size += com.google.protobuf.CodedOutputStream
@@ -4306,14 +4363,14 @@ public final class ProtosFactory {
       memoizedSerializedSize = size;
       return size;
     }
-
+    
     private static final long serialVersionUID = 0L;
     @java.lang.Override
     protected java.lang.Object writeReplace()
         throws java.io.ObjectStreamException {
       return super.writeReplace();
     }
-
+    
     public static org.apache.flume.channel.file.proto.ProtosFactory.Commit parseFrom(
         com.google.protobuf.ByteString data)
        

<TRUNCATED>

[3/3] git commit: FLUME-1586: File Channel should support verifying integrity of individual events.

Posted by br...@apache.org.
FLUME-1586: File Channel should support verifying integrity of individual events.

(Hari Shreedharan via Brock Noland)


Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/c57ebd1d
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/c57ebd1d
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/c57ebd1d

Branch: refs/heads/trunk
Commit: c57ebd1d2296fd1c0c8f84eecb16799de2a73e35
Parents: 791f443
Author: Brock Noland <br...@apache.org>
Authored: Wed Jun 12 17:35:38 2013 -0700
Committer: Brock Noland <br...@apache.org>
Committed: Wed Jun 12 17:35:38 2013 -0700

----------------------------------------------------------------------
 bin/flume-ng                                    |    6 +
 .../channel/file/CorruptEventException.java     |   36 +
 .../apache/flume/channel/file/FileChannel.java  |   47 +-
 .../java/org/apache/flume/channel/file/Log.java |   12 +-
 .../org/apache/flume/channel/file/LogFile.java  |  104 +-
 .../apache/flume/channel/file/LogFileV3.java    |   18 +-
 .../flume/channel/file/NoopRecordException.java |   35 +
 .../java/org/apache/flume/channel/file/Put.java |   38 +-
 .../flume/channel/file/ReplayHandler.java       |    3 +-
 .../flume/channel/file/Serialization.java       |   15 +-
 .../channel/file/TransactionEventRecord.java    |   10 +-
 .../flume/channel/file/proto/ProtosFactory.java | 1127 +++++++++---------
 .../src/main/proto/filechannel.proto            |    1 +
 .../flume/channel/file/TestFileChannel.java     |   44 +-
 .../org/apache/flume/channel/file/TestLog.java  |   28 +-
 .../apache/flume/channel/file/TestLogFile.java  |  111 +-
 .../file/TestTransactionEventRecordV3.java      |   16 +-
 .../apache/flume/channel/file/TestUtils.java    |   55 +-
 flume-ng-dist/pom.xml                           |    4 +
 flume-ng-dist/src/main/assembly/bin.xml         |    1 +
 flume-ng-dist/src/main/assembly/src.xml         |    2 +
 flume-tools/pom.xml                             |  143 +++
 .../flume/tools/FileChannelIntegrityTool.java   |  142 +++
 .../java/org/apache/flume/tools/FlumeTool.java  |   24 +
 .../org/apache/flume/tools/FlumeToolType.java   |   40 +
 .../org/apache/flume/tools/FlumeToolsMain.java  |   68 ++
 .../tools/TestFileChannelIntegrityTool.java     |  247 ++++
 pom.xml                                         |    7 +
 28 files changed, 1765 insertions(+), 619 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/bin/flume-ng
----------------------------------------------------------------------
diff --git a/bin/flume-ng b/bin/flume-ng
index 22b95b8..65cc985 100755
--- a/bin/flume-ng
+++ b/bin/flume-ng
@@ -26,6 +26,7 @@
 FLUME_AGENT_CLASS="org.apache.flume.node.Application"
 FLUME_AVRO_CLIENT_CLASS="org.apache.flume.client.avro.AvroCLIClient"
 FLUME_VERSION_CLASS="org.apache.flume.tools.VersionInfo"
+FLUME_TOOLS_CLASS="org.apache.flume.tools.FlumeToolsMain"
 
 CLEAN_FLAG=1
 ################################
@@ -261,6 +262,9 @@ case "$mode" in
   avro-client)
     opt_avro_client=1
     ;;
+  tool)
+    opt_tool=1
+    ;;
   version)
    opt_version=1
    CLEAN_FLAG=0
@@ -433,6 +437,8 @@ elif [ -n "$opt_avro_client" ] ; then
   run_flume $FLUME_AVRO_CLIENT_CLASS $args
 elif [ -n "${opt_version}" ] ; then
   run_flume $FLUME_VERSION_CLASS $args
+elif [ -n "${opt_tool}" ] ; then
+  run_flume $FLUME_TOOLS_CLASS $args
 else
   error "This message should never appear" 1
 fi

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
new file mode 100644
index 0000000..691d291
--- /dev/null
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.channel.file;
+
+
+public class CorruptEventException extends Exception {
+
+  private static final long serialVersionUID = -2986946303540798416L;
+  public CorruptEventException() {
+    super();
+  }
+
+  public CorruptEventException(String msg) {
+    super(msg);
+  }
+
+  public CorruptEventException(String msg, Throwable th) {
+    super(msg, th);
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
index cc0d38a..36f150b 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
@@ -100,6 +100,7 @@ public class FileChannel extends BasicChannelSemantics {
   private String encryptionActiveKey;
   private String encryptionCipherProvider;
   private boolean useDualCheckpoints;
+  private boolean isTest = false;
 
   @Override
   public synchronized void setName(String name) {
@@ -451,6 +452,7 @@ public class FileChannel extends BasicChannelSemantics {
     private String getStateAsString() {
       return String.valueOf(getState());
     }
+
     @Override
     protected void doPut(Event event) throws InterruptedException {
       channelCounter.incrementEventPutAttemptCount();
@@ -511,23 +513,40 @@ public class FileChannel extends BasicChannelSemantics {
             + "log. Try increasing the log write timeout value. " +
             channelNameDescriptor);
       }
+
+      /*
+       * 1. Take an event which is in the queue.
+       * 2. If getting that event does not throw NoopRecordException,
+       *    then return it.
+       * 3. Else try to retrieve the next event from the queue
+       * 4. Repeat 2 and 3 until queue is empty or an event is returned.
+       */
+
       try {
-        FlumeEventPointer ptr = queue.removeHead(transactionID);
-        if(ptr != null) {
-          try {
-            // first add to takeList so that if write to disk
-            // fails rollback actually does it's work
-            Preconditions.checkState(takeList.offer(ptr), "takeList offer failed "
-                 + channelNameDescriptor);
-            log.take(transactionID, ptr); // write take to disk
-            Event event = log.get(ptr);
-            return event;
-          } catch (IOException e) {
-            throw new ChannelException("Take failed due to IO error "
-                    + channelNameDescriptor, e);
+        while (true) {
+          FlumeEventPointer ptr = queue.removeHead(transactionID);
+          if (ptr == null) {
+            return null;
+          } else {
+            try {
+              // first add to takeList so that if write to disk
+              // fails rollback actually does it's work
+              Preconditions.checkState(takeList.offer(ptr),
+                "takeList offer failed "
+                  + channelNameDescriptor);
+              log.take(transactionID, ptr); // write take to disk
+              Event event = log.get(ptr);
+              return event;
+            } catch (IOException e) {
+              throw new ChannelException("Take failed due to IO error "
+                + channelNameDescriptor, e);
+            } catch (NoopRecordException e) {
+              LOG.warn("Corrupt record replaced by File Channel Integrity " +
+                "tool found. Will retrieve next event", e);
+              takeList.remove(ptr);
+            }
           }
         }
-        return null;
       } finally {
         log.unlockShared();
       }

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
index 1918baa..8dc0ff8 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
@@ -73,7 +73,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-class Log {
+public class Log {
   public static final String PREFIX = "log-";
   private static final Logger LOGGER = LoggerFactory.getLogger(Log.class);
   private static final int MIN_NUM_LOGS = 2;
@@ -571,12 +571,18 @@ class Log {
    * @throws InterruptedException
    */
   FlumeEvent get(FlumeEventPointer pointer) throws IOException,
-  InterruptedException {
+    InterruptedException, NoopRecordException {
     Preconditions.checkState(open, "Log is closed");
     int id = pointer.getFileID();
     LogFile.RandomReader logFile = idLogFileMap.get(id);
     Preconditions.checkNotNull(logFile, "LogFile is null for id " + id);
-    return logFile.get(pointer.getOffset());
+    try {
+      return logFile.get(pointer.getOffset());
+    } catch (CorruptEventException ex) {
+      open = false;
+      throw new IOException("Corrupt event found. Please run File Channel " +
+        "Integrity tool.", ex);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
index d3db896..bb8ce1a 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
@@ -22,6 +22,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
+import org.apache.flume.ChannelException;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
 import org.apache.flume.channel.file.encryption.CipherProvider;
 import org.apache.flume.channel.file.encryption.KeyProvider;
 import org.apache.flume.tools.DirectMemoryUtils;
@@ -31,6 +34,7 @@ import org.slf4j.LoggerFactory;
 import javax.annotation.Nullable;
 import java.io.EOFException;
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
@@ -40,7 +44,9 @@ import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.atomic.AtomicLong;
 
-abstract class LogFile {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class LogFile {
 
   private static final Logger LOG = LoggerFactory
       .getLogger(LogFile.class);
@@ -54,8 +60,9 @@ abstract class LogFile {
   private static final ByteBuffer FILL = DirectMemoryUtils.
       allocate(1024 * 1024); // preallocation, 1MB
 
-  protected static final byte OP_RECORD = Byte.MAX_VALUE;
-  protected static final byte OP_EOF = Byte.MIN_VALUE;
+  public static final byte OP_RECORD = Byte.MAX_VALUE;
+  public static final byte OP_NOOP = (Byte.MAX_VALUE + Byte.MIN_VALUE)/2;
+  public static final byte OP_EOF = Byte.MIN_VALUE;
 
   static {
     for (int i = 0; i < FILL.capacity(); i++) {
@@ -63,6 +70,13 @@ abstract class LogFile {
     }
   }
 
+  protected static void skipRecord(RandomAccessFile fileHandle,
+    int offset) throws IOException {
+    fileHandle.seek(offset);
+    int length = fileHandle.readInt();
+    fileHandle.skipBytes(length);
+  }
+
   abstract static class MetaDataWriter {
     private final File file;
     private final int logFileID;
@@ -296,6 +310,48 @@ abstract class LogFile {
     }
   }
 
+  /**
+   * This is an class meant to be an internal Flume API,
+   * and can change at any time. Intended to be used only from File Channel  Integrity
+   * test tool. Not to be used for any other purpose.
+   */
+  public static class OperationRecordUpdater {
+    private final RandomAccessFile fileHandle;
+    private final File file;
+
+    public OperationRecordUpdater(File file) throws FileNotFoundException {
+      Preconditions.checkState(file.exists(), "File to update, " +
+        file.toString() + " does not exist.");
+      this.file = file;
+      fileHandle = new RandomAccessFile(file, "rw");
+    }
+
+    public void markRecordAsNoop(long offset) throws IOException {
+      // First ensure that the offset actually is an OP_RECORD. There is a
+      // small possibility that it still is OP_RECORD,
+      // but is not actually the beginning of a record. Is there anything we
+      // can do about it?
+      fileHandle.seek(offset);
+      byte byteRead = fileHandle.readByte();
+      Preconditions.checkState(byteRead == OP_RECORD || byteRead == OP_NOOP,
+        "Expected to read a record but the byte read indicates EOF");
+      fileHandle.seek(offset);
+      LOG.info("Marking event as " + OP_NOOP + " at " + offset + " for file " +
+        file.toString());
+      fileHandle.writeByte(OP_NOOP);
+    }
+
+    public void close() {
+      try {
+        fileHandle.getFD().sync();
+        fileHandle.close();
+      } catch (IOException e) {
+        LOG.error("Could not close file handle to file " +
+          fileHandle.toString(), e);
+      }
+    }
+  }
+
   static abstract class RandomReader {
     private final File file;
     private final BlockingQueue<RandomAccessFile> readFileHandles =
@@ -311,7 +367,7 @@ abstract class LogFile {
     }
 
     protected abstract TransactionEventRecord doGet(RandomAccessFile fileHandle)
-        throws IOException;
+        throws IOException, CorruptEventException;
 
     abstract int getVersion();
 
@@ -323,13 +379,18 @@ abstract class LogFile {
       return encryptionKeyProvider;
     }
 
-    FlumeEvent get(int offset) throws IOException, InterruptedException {
+    FlumeEvent get(int offset) throws IOException, InterruptedException,
+      CorruptEventException, NoopRecordException {
       Preconditions.checkState(open, "File closed");
       RandomAccessFile fileHandle = checkOut();
       boolean error = true;
       try {
         fileHandle.seek(offset);
         byte operation = fileHandle.readByte();
+        if(operation == OP_NOOP) {
+          throw new NoopRecordException("No op record found. Corrupt record " +
+            "may have been repaired by File Channel Integrity tool");
+        }
         Preconditions.checkState(operation == OP_RECORD,
             Integer.toHexString(operation));
         TransactionEventRecord record = doGet(fileHandle);
@@ -408,7 +469,7 @@ abstract class LogFile {
     }
   }
 
-  static abstract class SequentialReader {
+  public static abstract class SequentialReader {
 
     private final RandomAccessFile fileHandle;
     private final FileChannel fileChannel;
@@ -434,7 +495,7 @@ abstract class LogFile {
       fileHandle = new RandomAccessFile(file, "r");
       fileChannel = fileHandle.getChannel();
     }
-    abstract LogRecord doNext(int offset) throws IOException;
+    abstract LogRecord doNext(int offset) throws IOException, CorruptEventException;
 
     abstract int getVersion();
 
@@ -488,7 +549,7 @@ abstract class LogFile {
       }
     }
 
-    LogRecord next() throws IOException {
+    public LogRecord next() throws IOException, CorruptEventException {
       int offset = -1;
       try {
         long position = fileChannel.position();
@@ -499,14 +560,26 @@ abstract class LogFile {
         }
         offset = (int) position;
         Preconditions.checkState(offset >= 0);
-        byte operation = fileHandle.readByte();
-        if(operation != OP_RECORD) {
-          if(operation == OP_EOF) {
+        while (offset < fileHandle.length()) {
+          byte operation = fileHandle.readByte();
+          if (operation == OP_RECORD) {
+            break;
+          } else if (operation == OP_EOF) {
             LOG.info("Encountered EOF at " + offset + " in " + file);
+            return null;
+          } else if (operation == OP_NOOP) {
+            LOG.info("No op event found in file: " + file.toString() +
+              " at " + offset + ". Skipping event.");
+            skipRecord(fileHandle, offset + 1);
+            offset = (int) fileHandle.getFilePointer();
+            continue;
           } else {
             LOG.error("Encountered non op-record at " + offset + " " +
-                Integer.toHexString(operation) + " in " + file);
+              Integer.toHexString(operation) + " in " + file);
+            return null;
           }
+        }
+        if(offset >= fileHandle.length()) {
           return null;
         }
         return doNext(offset);
@@ -518,7 +591,10 @@ abstract class LogFile {
       }
     }
 
-    void close() {
+    public long getPosition() throws IOException {
+      return fileChannel.position();
+    }
+    public void close() {
       if(fileHandle != null) {
         try {
           fileHandle.close();
@@ -540,7 +616,7 @@ abstract class LogFile {
     return buffer;
   }
 
-  public static void main(String[] args) throws EOFException, IOException {
+  public static void main(String[] args) throws EOFException, IOException, CorruptEventException {
     File file = new File(args[0]);
     LogFile.SequentialReader reader = null;
     try {

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
index d9a2a9b..38f6ecb 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
@@ -21,6 +21,8 @@ package org.apache.flume.channel.file;
 import com.google.common.base.Preconditions;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.GeneratedMessage;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
 import org.apache.flume.channel.file.encryption.CipherProvider;
 import org.apache.flume.channel.file.encryption.CipherProviderFactory;
 import org.apache.flume.channel.file.encryption.KeyProvider;
@@ -43,7 +45,9 @@ import java.util.concurrent.LinkedBlockingDeque;
  * Represents a single data file on disk. Has methods to write,
  * read sequentially (replay), and read randomly (channel takes).
  */
-class LogFileV3 extends LogFile {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class LogFileV3 extends LogFile {
   protected static final Logger LOGGER =
       LoggerFactory.getLogger(LogFileV3.class);
 
@@ -267,7 +271,7 @@ class LogFileV3 extends LogFile {
     }
     @Override
     protected TransactionEventRecord doGet(RandomAccessFile fileHandle)
-        throws IOException {
+        throws IOException, CorruptEventException {
       // readers are opened right when the file is created and thus
       // empty. As such we wait to initialize until there is some
       // data before we we initialize
@@ -297,10 +301,11 @@ class LogFileV3 extends LogFile {
     }
   }
 
-  static class SequentialReader extends LogFile.SequentialReader {
+  public static class SequentialReader extends LogFile.SequentialReader {
     private CipherProvider.Decryptor decryptor;
-    SequentialReader(File file, @Nullable KeyProvider encryptionKeyProvider)
-        throws EOFException, IOException {
+
+    public SequentialReader(File file, @Nullable KeyProvider
+      encryptionKeyProvider) throws EOFException, IOException {
       super(file, encryptionKeyProvider);
       File metaDataFile = Serialization.getMetaDataFile(file);
       FileInputStream inputStream = new FileInputStream(metaDataFile);
@@ -344,8 +349,9 @@ class LogFileV3 extends LogFile {
     public int getVersion() {
       return Serialization.VERSION_3;
     }
+
     @Override
-    LogRecord doNext(int offset) throws IOException {
+    LogRecord doNext(int offset) throws IOException, CorruptEventException {
       byte[] buffer = readDelimitedBuffer(getFileHandle());
       if(decryptor != null) {
         buffer = decryptor.decrypt(buffer);

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
new file mode 100644
index 0000000..5f446b8
--- /dev/null
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.channel.file;
+
+public class NoopRecordException extends Exception {
+  private static final long serialVersionUID = -7394180633208889738L;
+
+  public NoopRecordException() {
+    super();
+  }
+
+  public NoopRecordException(String msg) {
+    super(msg);
+  }
+
+  public NoopRecordException(String msg, Throwable th) {
+    super(msg, th);
+  }
+}

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
index 4235a79..f08f024 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
@@ -24,7 +24,10 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Map;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.flume.channel.file.proto.ProtosFactory;
 
 import com.google.common.base.Preconditions;
@@ -36,13 +39,19 @@ import com.google.protobuf.ByteString;
  */
 class Put extends TransactionEventRecord {
   private FlumeEvent event;
+  // Should we move this to a higher level to not make multiple instances?
+  // Doing that might cause performance issues, since access to this would
+  // need to be synchronized (the whole reset-update-getValue cycle would
+  // need to be).
+  private final Checksum checksum = new CRC32();
 
+  @VisibleForTesting
   Put(Long transactionID, Long logWriteOrderID) {
-    super(transactionID, logWriteOrderID);
+    this(transactionID, logWriteOrderID, null);
   }
 
   Put(Long transactionID, Long logWriteOrderID, FlumeEvent event) {
-    this(transactionID, logWriteOrderID);
+    super(transactionID, logWriteOrderID);
     this.event = event;
   }
 
@@ -78,11 +87,14 @@ class Put extends TransactionEventRecord {
       }
     }
     eventBuilder.setBody(ByteString.copyFrom(event.getBody()));
-    putBuilder.setEvent(eventBuilder.build());
+    ProtosFactory.FlumeEvent protoEvent = eventBuilder.build();
+    putBuilder.setEvent(protoEvent);
+    putBuilder.setChecksum(calculateChecksum(event.getBody()));
     putBuilder.build().writeDelimitedTo(out);
   }
   @Override
-  void readProtos(InputStream in) throws IOException {
+  void readProtos(InputStream in) throws IOException,
+    CorruptEventException {
     ProtosFactory.Put put = Preconditions.checkNotNull(ProtosFactory.
         Put.parseDelimitedFrom(in), "Put cannot be null");
     Map<String, String> headers = Maps.newHashMap();
@@ -90,9 +102,25 @@ class Put extends TransactionEventRecord {
     for(ProtosFactory.FlumeEventHeader header : protosEvent.getHeadersList()) {
       headers.put(header.getKey(), header.getValue());
     }
+    byte[] eventBody = protosEvent.getBody().toByteArray();
+
+    if (put.hasChecksum()) {
+      long eventBodyChecksum = calculateChecksum(eventBody);
+      if (eventBodyChecksum != put.getChecksum()) {
+        throw new CorruptEventException("Expected checksum for event was " +
+          eventBodyChecksum + " but the checksum of the event is " + put.getChecksum());
+      }
+    }
     // TODO when we remove v2, remove FlumeEvent and use EventBuilder here
-    event = new FlumeEvent(headers, protosEvent.getBody().toByteArray());
+    event = new FlumeEvent(headers, eventBody);
+  }
+
+  protected long calculateChecksum(byte[] body) {
+    checksum.reset();
+    checksum.update(body, 0, body.length);
+    return checksum.getValue();
   }
+
   @Override
   public short getRecordType() {
     return Type.PUT.get();

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
index fc47b23..c8f5fdd 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
@@ -26,6 +26,7 @@ import com.google.common.collect.SetMultimap;
 import com.google.common.collect.Sets;
 import org.apache.commons.collections.MultiMap;
 import org.apache.commons.collections.map.MultiValueMap;
+import org.apache.flume.ChannelException;
 import org.apache.flume.channel.file.encryption.KeyProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -373,7 +374,7 @@ class ReplayHandler {
       }
     }
   }
-  private LogRecord next() throws IOException {
+  private LogRecord next() throws IOException, CorruptEventException {
     LogRecord resultLogRecord = logRecordBuffer.poll();
     if(resultLogRecord != null) {
       // there is more log records to read

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
index d6897e1..f8160d9 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
@@ -21,6 +21,8 @@ package org.apache.flume.channel.file;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import org.apache.commons.io.FileUtils;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,7 +35,9 @@ import java.io.RandomAccessFile;
 import java.util.Collections;
 import java.util.Set;
 
-class Serialization {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Serialization {
   private Serialization() {}
 
   static final long SIZE_OF_INT = 4;
@@ -43,9 +47,10 @@ class Serialization {
   static final int VERSION_2 = 2;
   static final int VERSION_3 = 3;
 
-  static final String METADATA_FILENAME = ".meta";
-  static final String METADATA_TMP_FILENAME = ".tmp";
-  static final String OLD_METADATA_FILENAME = METADATA_FILENAME + ".old";
+  public static final String METADATA_FILENAME = ".meta";
+  public static final String METADATA_TMP_FILENAME = ".tmp";
+  public static final String OLD_METADATA_FILENAME = METADATA_FILENAME +
+    ".old";
 
   // 64 K buffer to copy files.
   private static final int FILE_COPY_BUFFER_SIZE = 64 * 1024;
@@ -121,7 +126,7 @@ class Serialization {
    * @param to Destination file - this file should not exist
    * @return true if the copy was successful
    */
-  static boolean copyFile(File from, File to) throws IOException {
+  public static boolean copyFile(File from, File to) throws IOException {
     Preconditions.checkNotNull(from, "Source file is null, file copy failed.");
     Preconditions.checkNotNull(to, "Destination file is null, " +
       "file copy failed.");

http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
index 073042f..dda9b3f 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
@@ -29,6 +29,8 @@ import java.io.OutputStream;
 import java.lang.reflect.Constructor;
 import java.nio.ByteBuffer;
 
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
 import org.apache.flume.channel.file.proto.ProtosFactory;
 import org.apache.hadoop.io.Writable;
 import org.slf4j.Logger;
@@ -41,7 +43,9 @@ import com.google.common.collect.ImmutableMap;
 /**
  * Base class for records in data file: Put, Take, Rollback, Commit
  */
-abstract class TransactionEventRecord implements Writable {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class TransactionEventRecord implements Writable {
   private static final Logger LOG = LoggerFactory
       .getLogger(TransactionEventRecord.class);
   private final long transactionID;
@@ -63,7 +67,7 @@ abstract class TransactionEventRecord implements Writable {
 
   abstract void writeProtos(OutputStream out) throws IOException;
 
-  abstract void readProtos(InputStream in) throws IOException;
+  abstract void readProtos(InputStream in) throws IOException, CorruptEventException;
 
   long getLogWriteOrderID() {
     return logWriteOrderID;
@@ -187,7 +191,7 @@ abstract class TransactionEventRecord implements Writable {
 
 
   static TransactionEventRecord fromByteArray(byte[] buffer)
-      throws IOException {
+      throws IOException, CorruptEventException {
     ByteArrayInputStream in = new ByteArrayInputStream(buffer);
     try {
       ProtosFactory.TransactionEventHeader header = Preconditions.