You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by br...@apache.org on 2013/06/13 02:36:32 UTC
[1/3] FLUME-1586: File Channel should support verifying integrity of
individual events.
Updated Branches:
refs/heads/trunk 791f443fa -> c57ebd1d2
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto b/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
index 1e668d2..25520e8 100644
--- a/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
+++ b/flume-ng-channels/flume-file-channel/src/main/proto/filechannel.proto
@@ -56,6 +56,7 @@ message TransactionEventHeader {
message Put {
required FlumeEvent event = 1;
+ optional sfixed64 checksum = 2;
}
message Take {
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
index 0f7d14d..25765b5 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestFileChannel.java
@@ -21,13 +21,17 @@ package org.apache.flume.channel.file;
import static org.apache.flume.channel.file.TestUtils.*;
import static org.fest.reflect.core.Reflection.*;
+import java.io.File;
+import java.io.FilenameFilter;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
@@ -487,7 +491,7 @@ public class TestFileChannel extends TestFileChannelBase {
public void testReferenceCounts() throws Exception {
Map<String, String> overrides = Maps.newHashMap();
overrides.put(FileChannelConfiguration.CHECKPOINT_INTERVAL, "10000");
- overrides.put(FileChannelConfiguration.MAX_FILE_SIZE, "100");
+ overrides.put(FileChannelConfiguration.MAX_FILE_SIZE, "150");
final FileChannel channel = createFileChannel(overrides);
channel.start();
putEvents(channel, "testing-reference-counting", 1, 15);
@@ -568,4 +572,42 @@ public class TestFileChannel extends TestFileChannelBase {
}
+ @Test (expected = IllegalStateException.class)
+ public void testChannelDiesOnCorruptEvent() throws Exception {
+ final FileChannel channel = createFileChannel();
+ channel.start();
+ putEvents(channel,"test-corrupt-event",100,100);
+ for(File dataDir : dataDirs) {
+ File[] files = dataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if(!name.endsWith("meta") && !name.contains("lock")){
+ return true;
+ }
+ return false;
+ }
+ });
+ if (files != null && files.length > 0) {
+ for (int j = 0; j < files.length; j++) {
+ RandomAccessFile fileToCorrupt = new RandomAccessFile(files[0], "rw");
+ fileToCorrupt.seek(50);
+ fileToCorrupt.writeByte(234);
+ fileToCorrupt.close();
+ }
+ }
+ }
+ try {
+ consumeChannel(channel, true);
+ } catch (IllegalStateException ex) {
+ // The rollback call in takeEvents() in TestUtils will cause an
+ // IllegalArgumentException - and this should be tested to verify the
+ // channel is completely stopped.
+ Assert.assertTrue(ex.getMessage().contains("Log is closed"));
+ throw ex;
+ }
+ Assert.fail();
+
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
index 54978f8..8a5f8ad 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLog.java
@@ -74,7 +74,8 @@ public class TestLog {
* not transactional so the commit is not required.
*/
@Test
- public void testPutGet() throws IOException, InterruptedException {
+ public void testPutGet()
+ throws IOException, InterruptedException, NoopRecordException {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
long transactionID = ++this.transactionID;
FlumeEventPointer eventPointer = log.put(transactionID, eventIn);
@@ -86,7 +87,8 @@ public class TestLog {
Assert.assertArrayEquals(eventIn.getBody(), eventOut.getBody());
}
@Test
- public void testRoll() throws IOException, InterruptedException {
+ public void testRoll()
+ throws IOException, InterruptedException, NoopRecordException {
log.shutdownWorker();
Thread.sleep(1000);
for (int i = 0; i < 1000; i++) {
@@ -107,15 +109,16 @@ public class TestLog {
}
}
}
- // 78 (*2 for meta) files with TestLog.MAX_FILE_SIZE=1000
- Assert.assertEquals(156, logCount);
+ // 93 (*2 for meta) files with TestLog.MAX_FILE_SIZE=1000
+ Assert.assertEquals(186, logCount);
}
/**
* After replay of the log, we should find the event because the put
* was committed
*/
@Test
- public void testPutCommit() throws IOException, InterruptedException {
+ public void testPutCommit()
+ throws IOException, InterruptedException, NoopRecordException {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
long transactionID = ++this.transactionID;
FlumeEventPointer eventPointerIn = log.put(transactionID, eventIn);
@@ -243,16 +246,16 @@ public class TestLog {
*/
@Test
public void testPutTakeRollbackLogReplayV1()
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, NoopRecordException {
doPutTakeRollback(true);
}
@Test
public void testPutTakeRollbackLogReplayV2()
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, NoopRecordException {
doPutTakeRollback(false);
}
public void doPutTakeRollback(boolean useLogReplayV1)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, NoopRecordException {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
long putTransactionID = ++transactionID;
FlumeEventPointer eventPointerIn = log.put(putTransactionID, eventIn);
@@ -392,7 +395,7 @@ public class TestLog {
}
@Test
public void testReplaySucceedsWithUnusedEmptyLogMetaDataNormalReplay()
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, NoopRecordException {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
long transactionID = ++this.transactionID;
FlumeEventPointer eventPointer = log.put(transactionID, eventIn);
@@ -406,7 +409,7 @@ public class TestLog {
}
@Test
public void testReplaySucceedsWithUnusedEmptyLogMetaDataFastReplay()
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException, NoopRecordException {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
long transactionID = ++this.transactionID;
FlumeEventPointer eventPointer = log.put(transactionID, eventIn);
@@ -422,7 +425,7 @@ public class TestLog {
}
public void doTestReplaySucceedsWithUnusedEmptyLogMetaData(FlumeEvent eventIn,
FlumeEventPointer eventPointer) throws IOException,
- InterruptedException {
+ InterruptedException, NoopRecordException {
for (int i = 0; i < dataDirs.length; i++) {
for(File logFile : LogUtils.getLogs(dataDirs[i])) {
if(logFile.length() == 0L) {
@@ -461,7 +464,8 @@ public class TestLog {
}
private void takeAndVerify(FlumeEventPointer eventPointerIn,
- FlumeEvent eventIn) throws IOException, InterruptedException {
+ FlumeEvent eventIn)
+ throws IOException, InterruptedException, NoopRecordException {
FlumeEventQueue queue = log.getFlumeEventQueue();
FlumeEventPointer eventPointerOut = queue.removeHead(0);
Assert.assertNotNull(eventPointerOut);
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
index bef22ef..4da6ac1 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestLogFile.java
@@ -21,12 +21,16 @@ package org.apache.flume.channel.file;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -98,6 +102,8 @@ public class TestLogFile {
final List<Throwable> errors =
Collections.synchronizedList(new ArrayList<Throwable>());
ExecutorService executorService = Executors.newFixedThreadPool(10);
+ CompletionService<Void> completionService = new ExecutorCompletionService
+ <Void>(executorService);
final LogFile.RandomReader logFileReader =
LogFileFactory.getRandomReader(dataFile, null);
for (int i = 0; i < 1000; i++) {
@@ -117,7 +123,7 @@ public class TestLogFile {
ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
FlumeEventPointer ptr = logFileWriter.put(bytes);
final int offset = ptr.getOffset();
- executorService.submit(new Runnable() {
+ completionService.submit(new Runnable() {
@Override
public void run() {
try {
@@ -130,7 +136,11 @@ public class TestLogFile {
}
}
}
- });
+ }, null);
+ }
+
+ for(int i = 0; i < 1000; i++) {
+ completionService.take();
}
// first try and throw failures
for(Throwable throwable : errors) {
@@ -142,7 +152,8 @@ public class TestLogFile {
}
}
@Test
- public void testReader() throws InterruptedException, IOException {
+ public void testReader() throws InterruptedException, IOException,
+ CorruptEventException {
Map<Integer, Put> puts = Maps.newHashMap();
for (int i = 0; i < 1000; i++) {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
@@ -169,7 +180,8 @@ public class TestLogFile {
}
@Test
- public void testReaderOldMetaFile() throws InterruptedException, IOException {
+ public void testReaderOldMetaFile() throws InterruptedException,
+ IOException, CorruptEventException {
Map<Integer, Put> puts = Maps.newHashMap();
for (int i = 0; i < 1000; i++) {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
@@ -204,7 +216,8 @@ public class TestLogFile {
}
@Test
- public void testReaderTempMetaFile() throws InterruptedException, IOException {
+ public void testReaderTempMetaFile() throws InterruptedException,
+ IOException, CorruptEventException {
Map<Integer, Put> puts = Maps.newHashMap();
for (int i = 0; i < 1000; i++) {
FlumeEvent eventIn = TestUtils.newPersistableEvent();
@@ -260,4 +273,92 @@ public class TestLogFile {
Assert.assertEquals(3, metaData.getCheckpointPosition());
Assert.assertEquals(4, metaData.getCheckpointWriteOrderID());
}
+
+ @Test (expected = CorruptEventException.class)
+ public void testPutGetCorruptEvent() throws Exception {
+ final LogFile.RandomReader logFileReader =
+ LogFileFactory.getRandomReader(dataFile, null);
+ final FlumeEvent eventIn = TestUtils.newPersistableEvent(2500);
+ final Put put = new Put(++transactionID, WriteOrderOracle.next(),
+ eventIn);
+ ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
+ FlumeEventPointer ptr = logFileWriter.put(bytes);
+ logFileWriter.commit(TransactionEventRecord.toByteBuffer(new Commit
+ (transactionID, WriteOrderOracle.next())));
+ final int offset = ptr.getOffset();
+ RandomAccessFile writer = new RandomAccessFile(dataFile, "rw");
+ writer.seek(offset + 1500);
+ writer.write((byte) 45);
+ writer.write((byte) 12);
+ writer.getFD().sync();
+ logFileReader.get(offset);
+
+ // Should have thrown an exception by now.
+ Assert.fail();
+
+ }
+
+ @Test (expected = NoopRecordException.class)
+ public void testPutGetNoopEvent() throws Exception {
+ final LogFile.RandomReader logFileReader =
+ LogFileFactory.getRandomReader(dataFile, null);
+ final FlumeEvent eventIn = TestUtils.newPersistableEvent(2500);
+ final Put put = new Put(++transactionID, WriteOrderOracle.next(),
+ eventIn);
+ ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
+ FlumeEventPointer ptr = logFileWriter.put(bytes);
+ logFileWriter.commit(TransactionEventRecord.toByteBuffer(new Commit
+ (transactionID, WriteOrderOracle.next())));
+ final int offset = ptr.getOffset();
+ LogFile.OperationRecordUpdater updater = new LogFile
+ .OperationRecordUpdater(dataFile);
+ updater.markRecordAsNoop(offset);
+ logFileReader.get(offset);
+
+ // Should have thrown an exception by now.
+ Assert.fail();
+ }
+
+ @Test
+ public void testOperationRecordUpdater() throws Exception {
+ File tempDir = Files.createTempDir();
+ File temp = new File(tempDir, "temp");
+ final RandomAccessFile tempFile = new RandomAccessFile(temp, "rw");
+ for(int i = 0; i < 5000; i++) {
+ tempFile.write(LogFile.OP_RECORD);
+ }
+ tempFile.seek(0);
+ LogFile.OperationRecordUpdater recordUpdater = new LogFile
+ .OperationRecordUpdater(temp);
+ //Convert every 10th byte into a noop byte
+ for(int i = 0; i < 5000; i+=10) {
+ recordUpdater.markRecordAsNoop(i);
+ }
+ recordUpdater.close();
+
+ tempFile.seek(0);
+ // Verify every 10th byte is actually a NOOP
+ for(int i = 0; i < 5000; i+=10) {
+ tempFile.seek(i);
+ Assert.assertEquals(LogFile.OP_NOOP, tempFile.readByte());
+ }
+
+ }
+
+ @Test
+ public void testOpRecordUpdaterWithFlumeEvents() throws Exception{
+ final FlumeEvent eventIn = TestUtils.newPersistableEvent(2500);
+ final Put put = new Put(++transactionID, WriteOrderOracle.next(),
+ eventIn);
+ ByteBuffer bytes = TransactionEventRecord.toByteBuffer(put);
+ FlumeEventPointer ptr = logFileWriter.put(bytes);
+ logFileWriter.commit(TransactionEventRecord.toByteBuffer(new Commit
+ (transactionID, WriteOrderOracle.next())));
+ final int offset = ptr.getOffset();
+ LogFile.OperationRecordUpdater updater = new LogFile
+ .OperationRecordUpdater(dataFile);
+ updater.markRecordAsNoop(offset);
+ RandomAccessFile fileReader = new RandomAccessFile(dataFile, "rw");
+ Assert.assertEquals(LogFile.OP_NOOP, fileReader.readByte());
+ }
}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
index f403422..eb0ce04 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestTransactionEventRecordV3.java
@@ -53,7 +53,7 @@ public class TestTransactionEventRecordV3 {
commit.getRecordType());
}
@Test
- public void testPutSerialization() throws IOException {
+ public void testPutSerialization() throws IOException, CorruptEventException {
Map<String, String> headers = new HashMap<String, String>();
headers.put("key", "value");
Put in = new Put(System.currentTimeMillis(),
@@ -70,7 +70,8 @@ public class TestTransactionEventRecordV3 {
Assert.assertTrue(Arrays.equals(in.getEvent().getBody(), out.getEvent().getBody()));
}
@Test
- public void testPutSerializationNullHeader() throws IOException {
+ public void testPutSerializationNullHeader() throws IOException,
+ CorruptEventException {
Put in = new Put(System.currentTimeMillis(),
WriteOrderOracle.next(),
new FlumeEvent(null, new byte[0]));
@@ -84,7 +85,8 @@ public class TestTransactionEventRecordV3 {
Assert.assertTrue(Arrays.equals(in.getEvent().getBody(), out.getEvent().getBody()));
}
@Test
- public void testTakeSerialization() throws IOException {
+ public void testTakeSerialization() throws IOException,
+ CorruptEventException {
Take in = new Take(System.currentTimeMillis(),
WriteOrderOracle.next(), 10, 20);
Take out = (Take)TransactionEventRecord.fromByteArray(toByteArray(in));
@@ -97,7 +99,8 @@ public class TestTransactionEventRecordV3 {
}
@Test
- public void testRollbackSerialization() throws IOException {
+ public void testRollbackSerialization() throws IOException,
+ CorruptEventException {
Rollback in = new Rollback(System.currentTimeMillis(),
WriteOrderOracle.next());
Rollback out = (Rollback)TransactionEventRecord.fromByteArray(toByteArray(in));
@@ -108,7 +111,8 @@ public class TestTransactionEventRecordV3 {
}
@Test
- public void testCommitSerialization() throws IOException {
+ public void testCommitSerialization() throws IOException,
+ CorruptEventException {
Commit in = new Commit(System.currentTimeMillis(),
WriteOrderOracle.next());
Commit out = (Commit)TransactionEventRecord.fromByteArray(toByteArray(in));
@@ -119,7 +123,7 @@ public class TestTransactionEventRecordV3 {
}
@Test
- public void testBadType() throws IOException {
+ public void testBadType() throws IOException, CorruptEventException {
TransactionEventRecord in = mock(TransactionEventRecord.class);
when(in.getRecordType()).thenReturn(Short.MIN_VALUE);
try {
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
index 563dbcc..a5ab45a 100644
--- a/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
+++ b/flume-ng-channels/flume-file-channel/src/test/java/org/apache/flume/channel/file/TestUtils.java
@@ -29,6 +29,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -64,6 +65,16 @@ public class TestUtils {
return event;
}
+ public static FlumeEvent newPersistableEvent(int size) {
+ Map<String, String> headers = Maps.newHashMap();
+ String timestamp = String.valueOf(System.currentTimeMillis());
+ headers.put("timestamp", timestamp);
+ byte[] data = new byte[size];
+ Arrays.fill(data, (byte) 54);
+ FlumeEvent event = new FlumeEvent(headers, data);
+ return event;
+ }
+
public static DataInput toDataInput(Writable writable) throws IOException {
ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
DataOutputStream dataOutput = new DataOutputStream(byteOutput);
@@ -129,25 +140,47 @@ public class TestUtils {
.invoke(true));
}
+ public static Set<String> takeEvents(Channel channel, int batchSize)
+ throws Exception {
+ return takeEvents(channel, batchSize, false);
+ }
+
+ public static Set<String> takeEvents(Channel channel,
+ int batchSize, boolean checkForCorruption) throws Exception {
+ return takeEvents(channel, batchSize, Integer.MAX_VALUE, checkForCorruption);
+ }
+
public static Set<String> takeEvents(Channel channel,
- int batchSize) throws Exception {
- return takeEvents(channel, batchSize, Integer.MAX_VALUE);
+ int batchSize, int numEvents) throws Exception {
+ return takeEvents(channel, batchSize, numEvents, false);
}
public static Set<String> takeEvents(Channel channel,
- int batchSize, int numEvents) throws Exception {
+ int batchSize, int numEvents, boolean checkForCorruption) throws
+ Exception {
Set<String> result = Sets.newHashSet();
for (int i = 0; i < numEvents; i += batchSize) {
Transaction transaction = channel.getTransaction();
try {
transaction.begin();
for (int j = 0; j < batchSize; j++) {
- Event event = null;
+ Event event;
try {
event = channel.take();
} catch (ChannelException ex) {
- Assert.assertTrue(ex.getMessage().startsWith(
- "Take list for FileBackedTransaction, capacity"));
+ Throwable th = ex;
+ String msg;
+ if(checkForCorruption) {
+ msg = "Corrupt event found. Please run File Channel";
+ th = ex.getCause();
+ } else {
+ msg = "Take list for FileBackedTransaction, capacity";
+ }
+ Assert.assertTrue(th.getMessage().startsWith(
+ msg));
+ if(checkForCorruption) {
+ throw (Exception) th;
+ }
transaction.commit();
return result;
}
@@ -168,15 +201,19 @@ public class TestUtils {
}
return result;
}
- public static Set<String> consumeChannel(Channel channel)
- throws Exception {
+
+ public static Set<String> consumeChannel(Channel channel) throws Exception {
+ return consumeChannel(channel, false);
+ }
+ public static Set<String> consumeChannel(Channel channel,
+ boolean checkForCorruption) throws Exception {
Set<String> result = Sets.newHashSet();
int[] batchSizes = new int[] {
1000, 100, 10, 1
};
for (int i = 0; i < batchSizes.length; i++) {
while(true) {
- Set<String> batch = takeEvents(channel, batchSizes[i]);
+ Set<String> batch = takeEvents(channel, batchSizes[i], checkForCorruption);
if(batch.isEmpty()) {
break;
}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-dist/pom.xml
----------------------------------------------------------------------
diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml
index e266272..9a5f64e 100644
--- a/flume-ng-dist/pom.xml
+++ b/flume-ng-dist/pom.xml
@@ -137,6 +137,10 @@
<artifactId>flume-ng-log4jappender</artifactId>
<classifier>jar-with-dependencies</classifier>
</dependency>
+ <dependency>
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-tools</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-dist/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/flume-ng-dist/src/main/assembly/bin.xml b/flume-ng-dist/src/main/assembly/bin.xml
index 74f0608..925a54a 100644
--- a/flume-ng-dist/src/main/assembly/bin.xml
+++ b/flume-ng-dist/src/main/assembly/bin.xml
@@ -95,6 +95,7 @@
<exclude>flume-ng-legacy-sources/**</exclude>
<exclude>flume-ng-clients/**</exclude>
<exclude>flume-ng-embedded-agent/**</exclude>
+ <exclude>flume-tools/**</exclude>
<exclude>**/target/**</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-dist/src/main/assembly/src.xml
----------------------------------------------------------------------
diff --git a/flume-ng-dist/src/main/assembly/src.xml b/flume-ng-dist/src/main/assembly/src.xml
index 7fafab8..5b86994 100644
--- a/flume-ng-dist/src/main/assembly/src.xml
+++ b/flume-ng-dist/src/main/assembly/src.xml
@@ -47,6 +47,7 @@
<include>org.apache.flume:flume-ng-legacy-sources</include>
<include>org.apache.flume:flume-ng-clients</include>
<include>org.apache.flume:flume-ng-embedded-agent</include>
+ <include>org.apache.flume:flume-tools</include>
</includes>
<sources>
@@ -89,6 +90,7 @@
<exclude>flume-ng-legacy-sources/**</exclude>
<exclude>flume-ng-clients/**</exclude>
<exclude>flume-ng-embedded-agent/**</exclude>
+ <exclude>flume-tools/**</exclude>
<exclude>**/target/**</exclude>
<exclude>**/.classpath</exclude>
<exclude>**/.project</exclude>
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/pom.xml
----------------------------------------------------------------------
diff --git a/flume-tools/pom.xml b/flume-tools/pom.xml
new file mode 100644
index 0000000..386c766
--- /dev/null
+++ b/flume-tools/pom.xml
@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>flume-parent</artifactId>
+ <groupId>org.apache.flume</groupId>
+ <version>1.4.0-SNAPSHOT</version>
+ </parent>
+
+ <groupId>org.apache.flume</groupId>
+ <artifactId>flume-tools</artifactId>
+
+ <name>Flume NG Tools</name>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume.flume-ng-channels</groupId>
+ <artifactId>flume-file-channel</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.easytesting</groupId>
+ <artifactId>fest-reflect</artifactId>
+ <version>1.4</version>
+ </dependency>
+
+ </dependencies>
+ <profiles>
+
+ <profile>
+ <id>hadoop-1.0</id>
+ <activation>
+ <property>
+ <name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>1.0.1</hadoop.version>
+ <hadoop.common.artifact.id>hadoop-core</hadoop.common.artifact.id>
+ </properties>
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- required because the hadoop-core pom is missing these deps
+ and MiniDFSCluster pulls in the webhdfs classes -->
+ <dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-2</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>2.0.0-alpha</hadoop.version>
+ <hadoop.common.artifact.id>hadoop-common</hadoop.common.artifact.id>
+ </properties>
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+ </profile>
+
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java b/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
new file mode 100644
index 0000000..aa24fa5
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FileChannelIntegrityTool.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.flume.FlumeException;
+import org.apache.flume.channel.file.CorruptEventException;
+import org.apache.flume.channel.file.Log;
+import org.apache.flume.channel.file.LogFile;
+import org.apache.flume.channel.file.LogFileV3;
+import org.apache.flume.channel.file.LogRecord;
+import org.apache.flume.channel.file.Serialization;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class FileChannelIntegrityTool implements FlumeTool {
+ public static final Logger LOG = LoggerFactory.getLogger
+ (FileChannelIntegrityTool.class);
+
+ private final List<File> dataDirs = new ArrayList<File>();
+
+ @Override
+ public void run(String[] args) throws IOException, ParseException {
+ boolean shouldContinue = parseCommandLineOpts(args);
+ if(!shouldContinue) {
+ LOG.error("Could not parse command line options. Exiting ...");
+ System.exit(1);
+ }
+ for(File dataDir : dataDirs) {
+ File[] dataFiles = dataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if(!name.endsWith(Serialization.METADATA_FILENAME)
+ && !name.endsWith(Serialization.METADATA_TMP_FILENAME)
+ && !name.endsWith(Serialization.OLD_METADATA_FILENAME)
+ && !name.equals(Log.FILE_LOCK)) {
+ return true;
+ }
+ return false;
+ }
+ });
+ if (dataFiles != null && dataFiles.length > 0) {
+ for (File dataFile : dataFiles) {
+ LOG.info("Checking for corruption in " + dataFile.toString());
+ LogFile.SequentialReader reader =
+ new LogFileV3.SequentialReader(dataFile, null);
+ LogFile.OperationRecordUpdater updater = new LogFile
+ .OperationRecordUpdater(dataFile);
+ boolean fileDone = false;
+ boolean fileBackedup = false;
+ while (!fileDone) {
+ long eventPosition = 0;
+ try {
+ // This depends on the fact that events are of the form:
+ // Type, length, data.
+ eventPosition = reader.getPosition();
+ // Try to get the record, if the checksums don't match,
+ // this will throw a CorruptEventException - so the real logic
+ // is in the catch block below.
+ LogRecord record = reader.next();
+ if (record != null) {
+ record.getEvent();
+ } else {
+ fileDone = true;
+ }
+ } catch (CorruptEventException e) {
+ LOG.warn("Corruption found in " + dataFile.toString() + " at "
+ + eventPosition);
+ if (!fileBackedup) {
+ Serialization.copyFile(dataFile, new File(dataFile.getParent(),
+ dataFile.getName() + ".bak"));
+ fileBackedup = true;
+ }
+ updater.markRecordAsNoop(eventPosition);
+ }
+ }
+ updater.close();
+ reader.close();
+ }
+ }
+ }
+ }
+
+ private boolean parseCommandLineOpts(String[] args) throws ParseException {
+ Options options = new Options();
+ options
+ .addOption("l", "dataDirs", true, "Comma-separated list of data " +
+ "directories which the tool must verify. This option is mandatory")
+ .addOption("h", "help", false, "Display help");
+
+ CommandLineParser parser = new GnuParser();
+ CommandLine commandLine = parser.parse(options, args);
+ if(commandLine.hasOption("help")) {
+ new HelpFormatter().printHelp("java -jar fcintegritytool ",
+ options, true);
+ return false;
+ }
+ if(!commandLine.hasOption("dataDirs")) {
+ new HelpFormatter().printHelp("java -jar fcintegritytool ", "",
+ options, "dataDirs is required.", true);
+ return false;
+ } else {
+ String dataDirStr[] = commandLine.getOptionValue("dataDirs").split(",");
+ for(String dataDir : dataDirStr) {
+ File f = new File(dataDir);
+ if(!f.exists()) {
+ throw new FlumeException("Data directory, " + dataDir + " does not " +
+ "exist.");
+ }
+ dataDirs.add(f);
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java b/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
new file mode 100644
index 0000000..28da8fe
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FlumeTool.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+public interface FlumeTool {
+
+ public void run(String[] args) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
new file mode 100644
index 0000000..f886c89
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolType.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+public enum FlumeToolType {
+ FCINTEGRITYTOOL(FileChannelIntegrityTool.class);
+
+ private final Class<? extends FlumeTool> klass;
+ private FlumeToolType(Class<? extends FlumeTool> klass) {
+ this.klass = klass;
+ }
+
+ public Class<? extends FlumeTool> getClassInstance() {
+ return this.klass;
+ }
+
+ public static String getNames() {
+ StringBuilder builder = new StringBuilder();
+ for(FlumeToolType type: values()) {
+ builder.append(type.name().toLowerCase() + "\n");
+ }
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
new file mode 100644
index 0000000..799ce85
--- /dev/null
+++ b/flume-tools/src/main/java/org/apache/flume/tools/FlumeToolsMain.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Arrays;
+
+public class FlumeToolsMain implements FlumeTool {
+
+ // Does the actual work in the run method so we can test it without actually
+ // having to start another process.
+
+ public static void main(String[] args) throws Exception {
+ new FlumeToolsMain().run(args);
+ }
+
+ private FlumeToolsMain() {
+ //No op.
+ }
+
+ @Override
+ public void run(String[] args) throws Exception{
+ String error = "Expected name of tool and arguments for" +
+ " tool to be passed in on the command line. Please pass one of the " +
+ "following as arguments to this command: \n";
+ StringBuilder builder = new StringBuilder(error);
+ for(FlumeToolType type : FlumeToolType.values()) {
+ builder.append(type.name()).append("\n");
+ }
+ if(args == null || args.length == 0) {
+ System.out.println(builder.toString());
+ System.exit(1);
+ }
+ String toolName = args[0];
+ FlumeTool tool = null;
+ for(FlumeToolType type : FlumeToolType.values()) {
+ if(toolName.equalsIgnoreCase(type.name())) {
+ tool = type.getClassInstance().newInstance();
+ break;
+ }
+ }
+ Preconditions.checkNotNull(tool, "Cannot find tool matching " + toolName
+ + ". Please select one of: \n " + FlumeToolType.getNames());
+ if (args.length == 1) {
+ tool.run(new String[0]);
+ } else {
+ tool.run(Arrays.asList(args).subList(1, args.length).
+ toArray(new String[0]));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
----------------------------------------------------------------------
diff --git a/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java b/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
new file mode 100644
index 0000000..d328671
--- /dev/null
+++ b/flume-tools/src/test/java/org/apache/flume/tools/TestFileChannelIntegrityTool.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.tools;
+
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.flume.Context;
+import org.apache.flume.Event;
+import org.apache.flume.Transaction;
+import org.apache.flume.channel.file.FileChannel;
+import org.apache.flume.channel.file.FileChannelConfiguration;
+import org.apache.flume.channel.file.Log;
+import org.apache.flume.channel.file.LogFile;
+import org.apache.flume.channel.file.LogFileV3;
+import org.apache.flume.channel.file.LogRecord;
+import org.apache.flume.channel.file.Serialization;
+import org.apache.flume.channel.file.WriteOrderOracle;
+import org.apache.flume.event.EventBuilder;
+import org.junit.After;
+import org.junit.AfterClass;
+import static org.fest.reflect.core.Reflection.*;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.RandomAccessFile;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+
+public class TestFileChannelIntegrityTool {
+ private static File baseDir;
+ private static File origCheckpointDir;
+ private static File origDataDir;
+ private static Event event;
+ private static Context ctx;
+
+ private File checkpointDir;
+ private File dataDir;
+
+
+ @BeforeClass
+ public static void setUpClass() throws Exception{
+ createDataFiles();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ checkpointDir = new File(baseDir, "checkpoint");
+ dataDir = new File(baseDir, "dataDir");
+ Assert.assertTrue(checkpointDir.mkdirs() || checkpointDir.isDirectory());
+ Assert.assertTrue(dataDir.mkdirs() || dataDir.isDirectory());
+ File[] dataFiles = origDataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if(name.contains("lock")) {
+ return false;
+ }
+ return true;
+ }
+ });
+ for(File dataFile : dataFiles) {
+ Serialization.copyFile(dataFile, new File(dataDir, dataFile.getName()));
+ }
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ FileUtils.deleteDirectory(checkpointDir);
+ FileUtils.deleteDirectory(dataDir);
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ FileUtils.deleteDirectory(origCheckpointDir);
+ FileUtils.deleteDirectory(origDataDir);
+ }
+
+ @Test
+ public void testFixCorruptRecords() throws Exception {
+ doTestFixCorruptEvents(false);
+ }
+ @Test
+ public void testFixCorruptRecordsWithCheckpoint() throws Exception {
+ doTestFixCorruptEvents(true);
+ }
+
+ public void doTestFixCorruptEvents(boolean withCheckpoint) throws Exception {
+ Set<String> corruptFiles = new HashSet<String>();
+ File[] files = dataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if(name.contains("lock") || name.contains("meta")) {
+ return false;
+ }
+ return true;
+ }
+ });
+ Random random = new Random();
+ int corrupted = 0;
+ for (File dataFile : files) {
+ LogFile.SequentialReader reader =
+ new LogFileV3.SequentialReader(dataFile, null);
+ RandomAccessFile handle = new RandomAccessFile(dataFile, "rw");
+ long eventPosition1 = reader.getPosition();
+ LogRecord rec = reader.next();
+ //No point corrupting commits, so ignore them
+ if(rec == null ||
+ rec.getEvent().getClass().getName().
+ equals("org.apache.flume.channel.file.Commit")) {
+ handle.close();
+ reader.close();
+ continue;
+ }
+ long eventPosition2 = reader.getPosition();
+ rec = reader.next();
+ handle.seek(eventPosition1 + 100);
+ handle.writeInt(random.nextInt());
+ corrupted++;
+ corruptFiles.add(dataFile.getName());
+ if (rec == null ||
+ rec.getEvent().getClass().getName().
+ equals("org.apache.flume.channel.file.Commit")) {
+ handle.close();
+ reader.close();
+ continue;
+ }
+ handle.seek(eventPosition2 + 100);
+ handle.writeInt(random.nextInt());
+ corrupted++;
+ handle.close();
+ reader.close();
+
+ }
+ FileChannelIntegrityTool tool = new FileChannelIntegrityTool();
+ tool.run(new String[] {"-l", dataDir.toString()});
+ FileChannel channel = new FileChannel();
+ channel.setName("channel");
+ String cp;
+ if(withCheckpoint) {
+ cp = origCheckpointDir.toString();
+ } else {
+ FileUtils.deleteDirectory(checkpointDir);
+ Assert.assertTrue(checkpointDir.mkdirs());
+ cp = checkpointDir.toString();
+ }
+ ctx.put(FileChannelConfiguration.CHECKPOINT_DIR,cp);
+ ctx.put(FileChannelConfiguration.DATA_DIRS, dataDir.toString());
+ channel.configure(ctx);
+ channel.start();
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ int i = 0;
+ while(channel.take() != null) {
+ i++;
+ }
+ tx.commit();
+ tx.close();
+ channel.stop();
+ Assert.assertEquals(25 - corrupted, i);
+ files = dataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if(name.contains(".bak")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ Assert.assertEquals(corruptFiles.size(), files.length);
+ for(File file : files) {
+ String name = file.getName();
+ name = name.replaceAll(".bak", "");
+ Assert.assertTrue(corruptFiles.remove(name));
+ }
+ Assert.assertTrue(corruptFiles.isEmpty());
+ }
+
+ private static void createDataFiles() throws Exception {
+ final byte[] eventData = new byte[2000];
+ for(int i = 0; i < 2000; i++) {
+ eventData[i] = 1;
+ }
+ WriteOrderOracle.setSeed(System.currentTimeMillis());
+ event = EventBuilder.withBody(eventData);
+ baseDir = Files.createTempDir();
+ if(baseDir.exists()) {
+ FileUtils.deleteDirectory(baseDir);
+ }
+ baseDir = Files.createTempDir();
+ origCheckpointDir = new File(baseDir, "chkpt");
+ Assert.assertTrue(origCheckpointDir.mkdirs() || origCheckpointDir.isDirectory());
+ origDataDir = new File(baseDir, "data");
+ Assert.assertTrue(origDataDir.mkdirs() || origDataDir.isDirectory());
+ FileChannel channel = new FileChannel();
+ channel.setName("channel");
+ ctx = new Context();
+ ctx.put(FileChannelConfiguration.CAPACITY, "1000");
+ ctx.put(FileChannelConfiguration.CHECKPOINT_DIR, origCheckpointDir.toString());
+ ctx.put(FileChannelConfiguration.DATA_DIRS, origDataDir.toString());
+ ctx.put(FileChannelConfiguration.MAX_FILE_SIZE, "10000");
+ ctx.put(FileChannelConfiguration.TRANSACTION_CAPACITY, "100");
+ channel.configure(ctx);
+ channel.start();
+ for (int j = 0; j < 5; j++) {
+ Transaction tx = channel.getTransaction();
+ tx.begin();
+ for (int i = 0; i < 5; i++) {
+ channel.put(event);
+ }
+ tx.commit();
+ tx.close();
+ }
+ Log log = field("log")
+ .ofType(Log.class)
+ .in(channel)
+ .get();
+
+ Assert.assertTrue("writeCheckpoint returned false",
+ method("writeCheckpoint")
+ .withReturnType(Boolean.class)
+ .withParameterTypes(Boolean.class)
+ .in(log)
+ .invoke(true));
+ channel.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9982f5d..8026936 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,6 +63,7 @@ limitations under the License.
<module>flume-ng-clients</module>
<module>flume-ng-sdk</module>
<module>flume-ng-tests</module>
+ <module>flume-tools</module>
</modules>
<profiles>
@@ -888,6 +889,12 @@ limitations under the License.
<dependency>
<groupId>org.apache.flume</groupId>
+ <artifactId>flume-tools</artifactId>
+ <version>1.4.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flume</groupId>
<artifactId>flume-ng-node</artifactId>
<version>1.4.0-SNAPSHOT</version>
</dependency>
[2/3] FLUME-1586: File Channel should support verifying integrity of
individual events.
Posted by br...@apache.org.
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
index 4860ac2..6885832 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/proto/ProtosFactory.java
@@ -28,29 +28,29 @@ public final class ProtosFactory {
}
public interface CheckpointOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required sfixed32 version = 1;
boolean hasVersion();
int getVersion();
-
+
// required sfixed64 writeOrderID = 2;
boolean hasWriteOrderID();
long getWriteOrderID();
-
+
// required sfixed32 queueSize = 3;
boolean hasQueueSize();
int getQueueSize();
-
+
// required sfixed32 queueHead = 4;
boolean hasQueueHead();
int getQueueHead();
-
+
// repeated .ActiveLog activeLogs = 5;
- java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog>
+ java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog>
getActiveLogsList();
org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog getActiveLogs(int index);
int getActiveLogsCount();
- java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+ java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
getActiveLogsOrBuilderList();
org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder getActiveLogsOrBuilder(
int index);
@@ -63,26 +63,26 @@ public final class ProtosFactory {
super(builder);
}
private Checkpoint(boolean noInit) {}
-
+
private static final Checkpoint defaultInstance;
public static Checkpoint getDefaultInstance() {
return defaultInstance;
}
-
+
public Checkpoint getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_fieldAccessorTable;
}
-
+
private int bitField0_;
// required sfixed32 version = 1;
public static final int VERSION_FIELD_NUMBER = 1;
@@ -93,7 +93,7 @@ public final class ProtosFactory {
public int getVersion() {
return version_;
}
-
+
// required sfixed64 writeOrderID = 2;
public static final int WRITEORDERID_FIELD_NUMBER = 2;
private long writeOrderID_;
@@ -103,7 +103,7 @@ public final class ProtosFactory {
public long getWriteOrderID() {
return writeOrderID_;
}
-
+
// required sfixed32 queueSize = 3;
public static final int QUEUESIZE_FIELD_NUMBER = 3;
private int queueSize_;
@@ -113,7 +113,7 @@ public final class ProtosFactory {
public int getQueueSize() {
return queueSize_;
}
-
+
// required sfixed32 queueHead = 4;
public static final int QUEUEHEAD_FIELD_NUMBER = 4;
private int queueHead_;
@@ -123,14 +123,14 @@ public final class ProtosFactory {
public int getQueueHead() {
return queueHead_;
}
-
+
// repeated .ActiveLog activeLogs = 5;
public static final int ACTIVELOGS_FIELD_NUMBER = 5;
private java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> activeLogs_;
public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> getActiveLogsList() {
return activeLogs_;
}
- public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+ public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
getActiveLogsOrBuilderList() {
return activeLogs_;
}
@@ -144,7 +144,7 @@ public final class ProtosFactory {
int index) {
return activeLogs_.get(index);
}
-
+
private void initFields() {
version_ = 0;
writeOrderID_ = 0L;
@@ -156,7 +156,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasVersion()) {
memoizedIsInitialized = 0;
return false;
@@ -182,7 +182,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -203,12 +203,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -234,14 +234,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -308,14 +308,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -329,17 +329,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Checkpoint_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -352,7 +352,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
version_ = 0;
@@ -371,20 +371,20 @@ public final class ProtosFactory {
}
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint build() {
org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint result = buildPartial();
if (!result.isInitialized()) {
@@ -392,7 +392,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint result = buildPartial();
@@ -402,7 +402,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint result = new org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint(this);
int from_bitField0_ = bitField0_;
@@ -436,7 +436,7 @@ public final class ProtosFactory {
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint)other);
@@ -445,7 +445,7 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.Checkpoint.getDefaultInstance()) return this;
if (other.hasVersion()) {
@@ -478,7 +478,7 @@ public final class ProtosFactory {
activeLogsBuilder_ = null;
activeLogs_ = other.activeLogs_;
bitField0_ = (bitField0_ & ~0x00000010);
- activeLogsBuilder_ =
+ activeLogsBuilder_ =
com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
getActiveLogsFieldBuilder() : null;
} else {
@@ -489,33 +489,33 @@ public final class ProtosFactory {
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasVersion()) {
-
+
return false;
}
if (!hasWriteOrderID()) {
-
+
return false;
}
if (!hasQueueSize()) {
-
+
return false;
}
if (!hasQueueHead()) {
-
+
return false;
}
for (int i = 0; i < getActiveLogsCount(); i++) {
if (!getActiveLogs(i).isInitialized()) {
-
+
return false;
}
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -568,9 +568,9 @@ public final class ProtosFactory {
}
}
}
-
+
private int bitField0_;
-
+
// required sfixed32 version = 1;
private int version_ ;
public boolean hasVersion() {
@@ -591,7 +591,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed64 writeOrderID = 2;
private long writeOrderID_ ;
public boolean hasWriteOrderID() {
@@ -612,7 +612,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed32 queueSize = 3;
private int queueSize_ ;
public boolean hasQueueSize() {
@@ -633,7 +633,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed32 queueHead = 4;
private int queueHead_ ;
public boolean hasQueueHead() {
@@ -654,7 +654,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// repeated .ActiveLog activeLogs = 5;
private java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> activeLogs_ =
java.util.Collections.emptyList();
@@ -664,10 +664,10 @@ public final class ProtosFactory {
bitField0_ |= 0x00000010;
}
}
-
+
private com.google.protobuf.RepeatedFieldBuilder<
org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder> activeLogsBuilder_;
-
+
public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog> getActiveLogsList() {
if (activeLogsBuilder_ == null) {
return java.util.Collections.unmodifiableList(activeLogs_);
@@ -805,7 +805,7 @@ public final class ProtosFactory {
return activeLogsBuilder_.getMessageOrBuilder(index);
}
}
- public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+ public java.util.List<? extends org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
getActiveLogsOrBuilderList() {
if (activeLogsBuilder_ != null) {
return activeLogsBuilder_.getMessageOrBuilderList();
@@ -822,12 +822,12 @@ public final class ProtosFactory {
return getActiveLogsFieldBuilder().addBuilder(
index, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDefaultInstance());
}
- public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder>
+ public java.util.List<org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder>
getActiveLogsBuilderList() {
return getActiveLogsFieldBuilder().getBuilderList();
}
private com.google.protobuf.RepeatedFieldBuilder<
- org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
+ org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.Builder, org.apache.flume.channel.file.proto.ProtosFactory.ActiveLogOrBuilder>
getActiveLogsFieldBuilder() {
if (activeLogsBuilder_ == null) {
activeLogsBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
@@ -840,25 +840,25 @@ public final class ProtosFactory {
}
return activeLogsBuilder_;
}
-
+
// @@protoc_insertion_point(builder_scope:Checkpoint)
}
-
+
static {
defaultInstance = new Checkpoint(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:Checkpoint)
}
-
+
public interface ActiveLogOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required sfixed32 logFileID = 1;
boolean hasLogFileID();
int getLogFileID();
-
+
// required sfixed32 count = 2;
boolean hasCount();
int getCount();
@@ -871,26 +871,26 @@ public final class ProtosFactory {
super(builder);
}
private ActiveLog(boolean noInit) {}
-
+
private static final ActiveLog defaultInstance;
public static ActiveLog getDefaultInstance() {
return defaultInstance;
}
-
+
public ActiveLog getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_fieldAccessorTable;
}
-
+
private int bitField0_;
// required sfixed32 logFileID = 1;
public static final int LOGFILEID_FIELD_NUMBER = 1;
@@ -901,7 +901,7 @@ public final class ProtosFactory {
public int getLogFileID() {
return logFileID_;
}
-
+
// required sfixed32 count = 2;
public static final int COUNT_FIELD_NUMBER = 2;
private int count_;
@@ -911,7 +911,7 @@ public final class ProtosFactory {
public int getCount() {
return count_;
}
-
+
private void initFields() {
logFileID_ = 0;
count_ = 0;
@@ -920,7 +920,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasLogFileID()) {
memoizedIsInitialized = 0;
return false;
@@ -932,7 +932,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -944,12 +944,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -963,14 +963,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -1037,14 +1037,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -1058,17 +1058,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_ActiveLog_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -1080,7 +1080,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
logFileID_ = 0;
@@ -1089,20 +1089,20 @@ public final class ProtosFactory {
bitField0_ = (bitField0_ & ~0x00000002);
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog build() {
org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog result = buildPartial();
if (!result.isInitialized()) {
@@ -1110,7 +1110,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog result = buildPartial();
@@ -1120,7 +1120,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog result = new org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog(this);
int from_bitField0_ = bitField0_;
@@ -1137,7 +1137,7 @@ public final class ProtosFactory {
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog)other);
@@ -1146,7 +1146,7 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.ActiveLog.getDefaultInstance()) return this;
if (other.hasLogFileID()) {
@@ -1158,19 +1158,19 @@ public final class ProtosFactory {
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasLogFileID()) {
-
+
return false;
}
if (!hasCount()) {
-
+
return false;
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -1207,9 +1207,9 @@ public final class ProtosFactory {
}
}
}
-
+
private int bitField0_;
-
+
// required sfixed32 logFileID = 1;
private int logFileID_ ;
public boolean hasLogFileID() {
@@ -1230,7 +1230,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed32 count = 2;
private int count_ ;
public boolean hasCount() {
@@ -1251,46 +1251,46 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// @@protoc_insertion_point(builder_scope:ActiveLog)
}
-
+
static {
defaultInstance = new ActiveLog(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:ActiveLog)
}
-
+
public interface LogFileMetaDataOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required sfixed32 version = 1;
boolean hasVersion();
int getVersion();
-
+
// required sfixed32 logFileID = 2;
boolean hasLogFileID();
int getLogFileID();
-
+
// required sfixed64 checkpointPosition = 3;
boolean hasCheckpointPosition();
long getCheckpointPosition();
-
+
// required sfixed64 checkpointWriteOrderID = 4;
boolean hasCheckpointWriteOrderID();
long getCheckpointWriteOrderID();
-
+
// optional .LogFileEncryption encryption = 5;
boolean hasEncryption();
org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption getEncryption();
org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder getEncryptionOrBuilder();
-
+
// optional sfixed64 backupCheckpointPosition = 6;
boolean hasBackupCheckpointPosition();
long getBackupCheckpointPosition();
-
+
// optional sfixed64 backupCheckpointWriteOrderID = 7;
boolean hasBackupCheckpointWriteOrderID();
long getBackupCheckpointWriteOrderID();
@@ -1303,26 +1303,26 @@ public final class ProtosFactory {
super(builder);
}
private LogFileMetaData(boolean noInit) {}
-
+
private static final LogFileMetaData defaultInstance;
public static LogFileMetaData getDefaultInstance() {
return defaultInstance;
}
-
+
public LogFileMetaData getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_fieldAccessorTable;
}
-
+
private int bitField0_;
// required sfixed32 version = 1;
public static final int VERSION_FIELD_NUMBER = 1;
@@ -1333,7 +1333,7 @@ public final class ProtosFactory {
public int getVersion() {
return version_;
}
-
+
// required sfixed32 logFileID = 2;
public static final int LOGFILEID_FIELD_NUMBER = 2;
private int logFileID_;
@@ -1343,7 +1343,7 @@ public final class ProtosFactory {
public int getLogFileID() {
return logFileID_;
}
-
+
// required sfixed64 checkpointPosition = 3;
public static final int CHECKPOINTPOSITION_FIELD_NUMBER = 3;
private long checkpointPosition_;
@@ -1353,7 +1353,7 @@ public final class ProtosFactory {
public long getCheckpointPosition() {
return checkpointPosition_;
}
-
+
// required sfixed64 checkpointWriteOrderID = 4;
public static final int CHECKPOINTWRITEORDERID_FIELD_NUMBER = 4;
private long checkpointWriteOrderID_;
@@ -1363,7 +1363,7 @@ public final class ProtosFactory {
public long getCheckpointWriteOrderID() {
return checkpointWriteOrderID_;
}
-
+
// optional .LogFileEncryption encryption = 5;
public static final int ENCRYPTION_FIELD_NUMBER = 5;
private org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption encryption_;
@@ -1376,7 +1376,7 @@ public final class ProtosFactory {
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder getEncryptionOrBuilder() {
return encryption_;
}
-
+
// optional sfixed64 backupCheckpointPosition = 6;
public static final int BACKUPCHECKPOINTPOSITION_FIELD_NUMBER = 6;
private long backupCheckpointPosition_;
@@ -1386,7 +1386,7 @@ public final class ProtosFactory {
public long getBackupCheckpointPosition() {
return backupCheckpointPosition_;
}
-
+
// optional sfixed64 backupCheckpointWriteOrderID = 7;
public static final int BACKUPCHECKPOINTWRITEORDERID_FIELD_NUMBER = 7;
private long backupCheckpointWriteOrderID_;
@@ -1396,7 +1396,7 @@ public final class ProtosFactory {
public long getBackupCheckpointWriteOrderID() {
return backupCheckpointWriteOrderID_;
}
-
+
private void initFields() {
version_ = 0;
logFileID_ = 0;
@@ -1410,7 +1410,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasVersion()) {
memoizedIsInitialized = 0;
return false;
@@ -1436,7 +1436,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -1463,12 +1463,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -1502,14 +1502,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -1576,14 +1576,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -1597,17 +1597,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileMetaData_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -1620,7 +1620,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
version_ = 0;
@@ -1643,20 +1643,20 @@ public final class ProtosFactory {
bitField0_ = (bitField0_ & ~0x00000040);
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData build() {
org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData result = buildPartial();
if (!result.isInitialized()) {
@@ -1664,7 +1664,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData result = buildPartial();
@@ -1674,7 +1674,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData result = new org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData(this);
int from_bitField0_ = bitField0_;
@@ -1715,7 +1715,7 @@ public final class ProtosFactory {
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData)other);
@@ -1724,7 +1724,7 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.LogFileMetaData.getDefaultInstance()) return this;
if (other.hasVersion()) {
@@ -1751,33 +1751,33 @@ public final class ProtosFactory {
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasVersion()) {
-
+
return false;
}
if (!hasLogFileID()) {
-
+
return false;
}
if (!hasCheckpointPosition()) {
-
+
return false;
}
if (!hasCheckpointWriteOrderID()) {
-
+
return false;
}
if (hasEncryption()) {
if (!getEncryption().isInitialized()) {
-
+
return false;
}
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -1843,9 +1843,9 @@ public final class ProtosFactory {
}
}
}
-
+
private int bitField0_;
-
+
// required sfixed32 version = 1;
private int version_ ;
public boolean hasVersion() {
@@ -1866,7 +1866,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed32 logFileID = 2;
private int logFileID_ ;
public boolean hasLogFileID() {
@@ -1887,7 +1887,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed64 checkpointPosition = 3;
private long checkpointPosition_ ;
public boolean hasCheckpointPosition() {
@@ -1908,7 +1908,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed64 checkpointWriteOrderID = 4;
private long checkpointWriteOrderID_ ;
public boolean hasCheckpointWriteOrderID() {
@@ -1929,7 +1929,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// optional .LogFileEncryption encryption = 5;
private org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption encryption_ = org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDefaultInstance();
private com.google.protobuf.SingleFieldBuilder<
@@ -2007,7 +2007,7 @@ public final class ProtosFactory {
}
}
private com.google.protobuf.SingleFieldBuilder<
- org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.Builder, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder>
+ org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.Builder, org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryptionOrBuilder>
getEncryptionFieldBuilder() {
if (encryptionBuilder_ == null) {
encryptionBuilder_ = new com.google.protobuf.SingleFieldBuilder<
@@ -2019,7 +2019,7 @@ public final class ProtosFactory {
}
return encryptionBuilder_;
}
-
+
// optional sfixed64 backupCheckpointPosition = 6;
private long backupCheckpointPosition_ ;
public boolean hasBackupCheckpointPosition() {
@@ -2040,7 +2040,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// optional sfixed64 backupCheckpointWriteOrderID = 7;
private long backupCheckpointWriteOrderID_ ;
public boolean hasBackupCheckpointWriteOrderID() {
@@ -2061,29 +2061,29 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// @@protoc_insertion_point(builder_scope:LogFileMetaData)
}
-
+
static {
defaultInstance = new LogFileMetaData(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:LogFileMetaData)
}
-
+
public interface LogFileEncryptionOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required string cipherProvider = 1;
boolean hasCipherProvider();
String getCipherProvider();
-
+
// required string keyAlias = 2;
boolean hasKeyAlias();
String getKeyAlias();
-
+
// optional bytes parameters = 3;
boolean hasParameters();
com.google.protobuf.ByteString getParameters();
@@ -2096,26 +2096,26 @@ public final class ProtosFactory {
super(builder);
}
private LogFileEncryption(boolean noInit) {}
-
+
private static final LogFileEncryption defaultInstance;
public static LogFileEncryption getDefaultInstance() {
return defaultInstance;
}
-
+
public LogFileEncryption getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_fieldAccessorTable;
}
-
+
private int bitField0_;
// required string cipherProvider = 1;
public static final int CIPHERPROVIDER_FIELD_NUMBER = 1;
@@ -2128,7 +2128,7 @@ public final class ProtosFactory {
if (ref instanceof String) {
return (String) ref;
} else {
- com.google.protobuf.ByteString bs =
+ com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (com.google.protobuf.Internal.isValidUtf8(bs)) {
@@ -2140,7 +2140,7 @@ public final class ProtosFactory {
private com.google.protobuf.ByteString getCipherProviderBytes() {
java.lang.Object ref = cipherProvider_;
if (ref instanceof String) {
- com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8((String) ref);
cipherProvider_ = b;
return b;
@@ -2148,7 +2148,7 @@ public final class ProtosFactory {
return (com.google.protobuf.ByteString) ref;
}
}
-
+
// required string keyAlias = 2;
public static final int KEYALIAS_FIELD_NUMBER = 2;
private java.lang.Object keyAlias_;
@@ -2160,7 +2160,7 @@ public final class ProtosFactory {
if (ref instanceof String) {
return (String) ref;
} else {
- com.google.protobuf.ByteString bs =
+ com.google.protobuf.ByteString bs =
(com.google.protobuf.ByteString) ref;
String s = bs.toStringUtf8();
if (com.google.protobuf.Internal.isValidUtf8(bs)) {
@@ -2172,7 +2172,7 @@ public final class ProtosFactory {
private com.google.protobuf.ByteString getKeyAliasBytes() {
java.lang.Object ref = keyAlias_;
if (ref instanceof String) {
- com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString b =
com.google.protobuf.ByteString.copyFromUtf8((String) ref);
keyAlias_ = b;
return b;
@@ -2180,7 +2180,7 @@ public final class ProtosFactory {
return (com.google.protobuf.ByteString) ref;
}
}
-
+
// optional bytes parameters = 3;
public static final int PARAMETERS_FIELD_NUMBER = 3;
private com.google.protobuf.ByteString parameters_;
@@ -2190,7 +2190,7 @@ public final class ProtosFactory {
public com.google.protobuf.ByteString getParameters() {
return parameters_;
}
-
+
private void initFields() {
cipherProvider_ = "";
keyAlias_ = "";
@@ -2200,7 +2200,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasCipherProvider()) {
memoizedIsInitialized = 0;
return false;
@@ -2212,7 +2212,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -2227,12 +2227,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -2250,14 +2250,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -2324,14 +2324,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -2345,17 +2345,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_LogFileEncryption_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -2367,7 +2367,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
cipherProvider_ = "";
@@ -2378,20 +2378,20 @@ public final class ProtosFactory {
bitField0_ = (bitField0_ & ~0x00000004);
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption build() {
org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption result = buildPartial();
if (!result.isInitialized()) {
@@ -2399,7 +2399,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption result = buildPartial();
@@ -2409,7 +2409,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption result = new org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption(this);
int from_bitField0_ = bitField0_;
@@ -2430,7 +2430,7 @@ public final class ProtosFactory {
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption)other);
@@ -2439,7 +2439,7 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.LogFileEncryption.getDefaultInstance()) return this;
if (other.hasCipherProvider()) {
@@ -2454,19 +2454,19 @@ public final class ProtosFactory {
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasCipherProvider()) {
-
+
return false;
}
if (!hasKeyAlias()) {
-
+
return false;
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -2508,9 +2508,9 @@ public final class ProtosFactory {
}
}
}
-
+
private int bitField0_;
-
+
// required string cipherProvider = 1;
private java.lang.Object cipherProvider_ = "";
public boolean hasCipherProvider() {
@@ -2546,7 +2546,7 @@ public final class ProtosFactory {
cipherProvider_ = value;
onChanged();
}
-
+
// required string keyAlias = 2;
private java.lang.Object keyAlias_ = "";
public boolean hasKeyAlias() {
@@ -2582,7 +2582,7 @@ public final class ProtosFactory {
keyAlias_ = value;
onChanged();
}
-
+
// optional bytes parameters = 3;
private com.google.protobuf.ByteString parameters_ = com.google.protobuf.ByteString.EMPTY;
public boolean hasParameters() {
@@ -2606,29 +2606,29 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// @@protoc_insertion_point(builder_scope:LogFileEncryption)
}
-
+
static {
defaultInstance = new LogFileEncryption(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:LogFileEncryption)
}
-
+
public interface TransactionEventHeaderOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required sfixed32 type = 1;
boolean hasType();
int getType();
-
+
// required sfixed64 transactionID = 2;
boolean hasTransactionID();
long getTransactionID();
-
+
// required sfixed64 writeOrderID = 3;
boolean hasWriteOrderID();
long getWriteOrderID();
@@ -2641,26 +2641,26 @@ public final class ProtosFactory {
super(builder);
}
private TransactionEventHeader(boolean noInit) {}
-
+
private static final TransactionEventHeader defaultInstance;
public static TransactionEventHeader getDefaultInstance() {
return defaultInstance;
}
-
+
public TransactionEventHeader getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_fieldAccessorTable;
}
-
+
private int bitField0_;
// required sfixed32 type = 1;
public static final int TYPE_FIELD_NUMBER = 1;
@@ -2671,7 +2671,7 @@ public final class ProtosFactory {
public int getType() {
return type_;
}
-
+
// required sfixed64 transactionID = 2;
public static final int TRANSACTIONID_FIELD_NUMBER = 2;
private long transactionID_;
@@ -2681,7 +2681,7 @@ public final class ProtosFactory {
public long getTransactionID() {
return transactionID_;
}
-
+
// required sfixed64 writeOrderID = 3;
public static final int WRITEORDERID_FIELD_NUMBER = 3;
private long writeOrderID_;
@@ -2691,7 +2691,7 @@ public final class ProtosFactory {
public long getWriteOrderID() {
return writeOrderID_;
}
-
+
private void initFields() {
type_ = 0;
transactionID_ = 0L;
@@ -2701,7 +2701,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasType()) {
memoizedIsInitialized = 0;
return false;
@@ -2717,7 +2717,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -2732,12 +2732,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -2755,14 +2755,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -2829,14 +2829,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -2850,17 +2850,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_TransactionEventHeader_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -2872,7 +2872,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
type_ = 0;
@@ -2883,20 +2883,20 @@ public final class ProtosFactory {
bitField0_ = (bitField0_ & ~0x00000004);
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader build() {
org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader result = buildPartial();
if (!result.isInitialized()) {
@@ -2904,7 +2904,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader result = buildPartial();
@@ -2914,7 +2914,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader result = new org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader(this);
int from_bitField0_ = bitField0_;
@@ -2935,7 +2935,7 @@ public final class ProtosFactory {
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader)other);
@@ -2944,7 +2944,7 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.TransactionEventHeader.getDefaultInstance()) return this;
if (other.hasType()) {
@@ -2959,23 +2959,23 @@ public final class ProtosFactory {
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasType()) {
-
+
return false;
}
if (!hasTransactionID()) {
-
+
return false;
}
if (!hasWriteOrderID()) {
-
+
return false;
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3017,9 +3017,9 @@ public final class ProtosFactory {
}
}
}
-
+
private int bitField0_;
-
+
// required sfixed32 type = 1;
private int type_ ;
public boolean hasType() {
@@ -3040,7 +3040,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed64 transactionID = 2;
private long transactionID_ ;
public boolean hasTransactionID() {
@@ -3061,7 +3061,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed64 writeOrderID = 3;
private long writeOrderID_ ;
public boolean hasWriteOrderID() {
@@ -3082,25 +3082,29 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// @@protoc_insertion_point(builder_scope:TransactionEventHeader)
}
-
+
static {
defaultInstance = new TransactionEventHeader(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:TransactionEventHeader)
}
-
+
public interface PutOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required .FlumeEvent event = 1;
boolean hasEvent();
org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent getEvent();
org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder getEventOrBuilder();
+
+ // optional sfixed64 checksum = 2;
+ boolean hasChecksum();
+ long getChecksum();
}
public static final class Put extends
com.google.protobuf.GeneratedMessage
@@ -3110,26 +3114,26 @@ public final class ProtosFactory {
super(builder);
}
private Put(boolean noInit) {}
-
+
private static final Put defaultInstance;
public static Put getDefaultInstance() {
return defaultInstance;
}
-
+
public Put getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_fieldAccessorTable;
}
-
+
private int bitField0_;
// required .FlumeEvent event = 1;
public static final int EVENT_FIELD_NUMBER = 1;
@@ -3143,15 +3147,26 @@ public final class ProtosFactory {
public org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder getEventOrBuilder() {
return event_;
}
-
+
+ // optional sfixed64 checksum = 2;
+ public static final int CHECKSUM_FIELD_NUMBER = 2;
+ private long checksum_;
+ public boolean hasChecksum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ public long getChecksum() {
+ return checksum_;
+ }
+
private void initFields() {
event_ = org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.getDefaultInstance();
+ checksum_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasEvent()) {
memoizedIsInitialized = 0;
return false;
@@ -3163,38 +3178,45 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
if (((bitField0_ & 0x00000001) == 0x00000001)) {
output.writeMessage(1, event_);
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeSFixed64(2, checksum_);
+ }
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(1, event_);
}
- size += getUnknownFields().getSerializedSize();
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeSFixed64Size(2, checksum_);
+ }
+ size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.Put parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -3261,14 +3283,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Put prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -3282,17 +3304,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Put_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.Put.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -3305,7 +3327,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
if (eventBuilder_ == null) {
@@ -3314,22 +3336,24 @@ public final class ProtosFactory {
eventBuilder_.clear();
}
bitField0_ = (bitField0_ & ~0x00000001);
+ checksum_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000002);
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Put.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Put getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Put.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Put build() {
org.apache.flume.channel.file.proto.ProtosFactory.Put result = buildPartial();
if (!result.isInitialized()) {
@@ -3337,7 +3361,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.Put buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.Put result = buildPartial();
@@ -3347,7 +3371,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Put buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.Put result = new org.apache.flume.channel.file.proto.ProtosFactory.Put(this);
int from_bitField0_ = bitField0_;
@@ -3360,11 +3384,15 @@ public final class ProtosFactory {
} else {
result.event_ = eventBuilder_.build();
}
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.checksum_ = checksum_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Put) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Put)other);
@@ -3373,28 +3401,31 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Put other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.Put.getDefaultInstance()) return this;
if (other.hasEvent()) {
mergeEvent(other.getEvent());
}
+ if (other.hasChecksum()) {
+ setChecksum(other.getChecksum());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasEvent()) {
-
+
return false;
}
if (!getEvent().isInitialized()) {
-
+
return false;
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3427,12 +3458,17 @@ public final class ProtosFactory {
setEvent(subBuilder.buildPartial());
break;
}
+ case 17: {
+ bitField0_ |= 0x00000002;
+ checksum_ = input.readSFixed64();
+ break;
+ }
}
}
}
-
+
private int bitField0_;
-
+
// required .FlumeEvent event = 1;
private org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent event_ = org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.getDefaultInstance();
private com.google.protobuf.SingleFieldBuilder<
@@ -3510,7 +3546,7 @@ public final class ProtosFactory {
}
}
private com.google.protobuf.SingleFieldBuilder<
- org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.Builder, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder>
+ org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEvent.Builder, org.apache.flume.channel.file.proto.ProtosFactory.FlumeEventOrBuilder>
getEventFieldBuilder() {
if (eventBuilder_ == null) {
eventBuilder_ = new com.google.protobuf.SingleFieldBuilder<
@@ -3522,25 +3558,46 @@ public final class ProtosFactory {
}
return eventBuilder_;
}
-
+
+ // optional sfixed64 checksum = 2;
+ private long checksum_ ;
+ public boolean hasChecksum() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ public long getChecksum() {
+ return checksum_;
+ }
+ public Builder setChecksum(long value) {
+ bitField0_ |= 0x00000002;
+ checksum_ = value;
+ onChanged();
+ return this;
+ }
+ public Builder clearChecksum() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ checksum_ = 0L;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:Put)
}
-
+
static {
defaultInstance = new Put(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:Put)
}
-
+
public interface TakeOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required sfixed32 fileID = 1;
boolean hasFileID();
int getFileID();
-
+
// required sfixed32 offset = 2;
boolean hasOffset();
int getOffset();
@@ -3553,26 +3610,26 @@ public final class ProtosFactory {
super(builder);
}
private Take(boolean noInit) {}
-
+
private static final Take defaultInstance;
public static Take getDefaultInstance() {
return defaultInstance;
}
-
+
public Take getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_fieldAccessorTable;
}
-
+
private int bitField0_;
// required sfixed32 fileID = 1;
public static final int FILEID_FIELD_NUMBER = 1;
@@ -3583,7 +3640,7 @@ public final class ProtosFactory {
public int getFileID() {
return fileID_;
}
-
+
// required sfixed32 offset = 2;
public static final int OFFSET_FIELD_NUMBER = 2;
private int offset_;
@@ -3593,7 +3650,7 @@ public final class ProtosFactory {
public int getOffset() {
return offset_;
}
-
+
private void initFields() {
fileID_ = 0;
offset_ = 0;
@@ -3602,7 +3659,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasFileID()) {
memoizedIsInitialized = 0;
return false;
@@ -3614,7 +3671,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -3626,12 +3683,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -3645,14 +3702,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.Take parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -3719,14 +3776,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Take prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -3740,17 +3797,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Take_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.Take.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -3762,7 +3819,7 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
fileID_ = 0;
@@ -3771,20 +3828,20 @@ public final class ProtosFactory {
bitField0_ = (bitField0_ & ~0x00000002);
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Take.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Take getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Take.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Take build() {
org.apache.flume.channel.file.proto.ProtosFactory.Take result = buildPartial();
if (!result.isInitialized()) {
@@ -3792,7 +3849,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.Take buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.Take result = buildPartial();
@@ -3802,7 +3859,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Take buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.Take result = new org.apache.flume.channel.file.proto.ProtosFactory.Take(this);
int from_bitField0_ = bitField0_;
@@ -3819,7 +3876,7 @@ public final class ProtosFactory {
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Take) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Take)other);
@@ -3828,7 +3885,7 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Take other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.Take.getDefaultInstance()) return this;
if (other.hasFileID()) {
@@ -3840,19 +3897,19 @@ public final class ProtosFactory {
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
if (!hasFileID()) {
-
+
return false;
}
if (!hasOffset()) {
-
+
return false;
}
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3889,9 +3946,9 @@ public final class ProtosFactory {
}
}
}
-
+
private int bitField0_;
-
+
// required sfixed32 fileID = 1;
private int fileID_ ;
public boolean hasFileID() {
@@ -3912,7 +3969,7 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// required sfixed32 offset = 2;
private int offset_ ;
public boolean hasOffset() {
@@ -3933,18 +3990,18 @@ public final class ProtosFactory {
onChanged();
return this;
}
-
+
// @@protoc_insertion_point(builder_scope:Take)
}
-
+
static {
defaultInstance = new Take(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:Take)
}
-
+
public interface RollbackOrBuilder
extends com.google.protobuf.MessageOrBuilder {
}
@@ -3956,61 +4013,61 @@ public final class ProtosFactory {
super(builder);
}
private Rollback(boolean noInit) {}
-
+
private static final Rollback defaultInstance;
public static Rollback getDefaultInstance() {
return defaultInstance;
}
-
+
public Rollback getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_fieldAccessorTable;
}
-
+
private void initFields() {
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.Rollback parseFrom(
com.google.protobuf.ByteString data)
throws com.google.protobuf.InvalidProtocolBufferException {
@@ -4077,14 +4134,14 @@ public final class ProtosFactory {
return newBuilder().mergeFrom(input, extensionRegistry)
.buildParsed();
}
-
+
public static Builder newBuilder() { return Builder.create(); }
public Builder newBuilderForType() { return newBuilder(); }
public static Builder newBuilder(org.apache.flume.channel.file.proto.ProtosFactory.Rollback prototype) {
return newBuilder().mergeFrom(prototype);
}
public Builder toBuilder() { return newBuilder(this); }
-
+
@java.lang.Override
protected Builder newBuilderForType(
com.google.protobuf.GeneratedMessage.BuilderParent parent) {
@@ -4098,17 +4155,17 @@ public final class ProtosFactory {
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Rollback_fieldAccessorTable;
}
-
+
// Construct using org.apache.flume.channel.file.proto.ProtosFactory.Rollback.newBuilder()
private Builder() {
maybeForceBuilderInitialization();
}
-
+
private Builder(BuilderParent parent) {
super(parent);
maybeForceBuilderInitialization();
@@ -4120,25 +4177,25 @@ public final class ProtosFactory {
private static Builder create() {
return new Builder();
}
-
+
public Builder clear() {
super.clear();
return this;
}
-
+
public Builder clone() {
return create().mergeFrom(buildPartial());
}
-
+
public com.google.protobuf.Descriptors.Descriptor
getDescriptorForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Rollback.getDescriptor();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Rollback getDefaultInstanceForType() {
return org.apache.flume.channel.file.proto.ProtosFactory.Rollback.getDefaultInstance();
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Rollback build() {
org.apache.flume.channel.file.proto.ProtosFactory.Rollback result = buildPartial();
if (!result.isInitialized()) {
@@ -4146,7 +4203,7 @@ public final class ProtosFactory {
}
return result;
}
-
+
private org.apache.flume.channel.file.proto.ProtosFactory.Rollback buildParsed()
throws com.google.protobuf.InvalidProtocolBufferException {
org.apache.flume.channel.file.proto.ProtosFactory.Rollback result = buildPartial();
@@ -4156,13 +4213,13 @@ public final class ProtosFactory {
}
return result;
}
-
+
public org.apache.flume.channel.file.proto.ProtosFactory.Rollback buildPartial() {
org.apache.flume.channel.file.proto.ProtosFactory.Rollback result = new org.apache.flume.channel.file.proto.ProtosFactory.Rollback(this);
onBuilt();
return result;
}
-
+
public Builder mergeFrom(com.google.protobuf.Message other) {
if (other instanceof org.apache.flume.channel.file.proto.ProtosFactory.Rollback) {
return mergeFrom((org.apache.flume.channel.file.proto.ProtosFactory.Rollback)other);
@@ -4171,17 +4228,17 @@ public final class ProtosFactory {
return this;
}
}
-
+
public Builder mergeFrom(org.apache.flume.channel.file.proto.ProtosFactory.Rollback other) {
if (other == org.apache.flume.channel.file.proto.ProtosFactory.Rollback.getDefaultInstance()) return this;
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
-
+
public final boolean isInitialized() {
return true;
}
-
+
public Builder mergeFrom(
com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -4208,22 +4265,22 @@ public final class ProtosFactory {
}
}
}
-
-
+
+
// @@protoc_insertion_point(builder_scope:Rollback)
}
-
+
static {
defaultInstance = new Rollback(true);
defaultInstance.initFields();
}
-
+
// @@protoc_insertion_point(class_scope:Rollback)
}
-
+
public interface CommitOrBuilder
extends com.google.protobuf.MessageOrBuilder {
-
+
// required sfixed32 type = 1;
boolean hasType();
int getType();
@@ -4236,26 +4293,26 @@ public final class ProtosFactory {
super(builder);
}
private Commit(boolean noInit) {}
-
+
private static final Commit defaultInstance;
public static Commit getDefaultInstance() {
return defaultInstance;
}
-
+
public Commit getDefaultInstanceForType() {
return defaultInstance;
}
-
+
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Commit_descriptor;
}
-
+
protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
internalGetFieldAccessorTable() {
return org.apache.flume.channel.file.proto.ProtosFactory.internal_static_Commit_fieldAccessorTable;
}
-
+
private int bitField0_;
// required sfixed32 type = 1;
public static final int TYPE_FIELD_NUMBER = 1;
@@ -4266,7 +4323,7 @@ public final class ProtosFactory {
public int getType() {
return type_;
}
-
+
private void initFields() {
type_ = 0;
}
@@ -4274,7 +4331,7 @@ public final class ProtosFactory {
public final boolean isInitialized() {
byte isInitialized = memoizedIsInitialized;
if (isInitialized != -1) return isInitialized == 1;
-
+
if (!hasType()) {
memoizedIsInitialized = 0;
return false;
@@ -4282,7 +4339,7 @@ public final class ProtosFactory {
memoizedIsInitialized = 1;
return true;
}
-
+
public void writeTo(com.google.protobuf.CodedOutputStream output)
throws java.io.IOException {
getSerializedSize();
@@ -4291,12 +4348,12 @@ public final class ProtosFactory {
}
getUnknownFields().writeTo(output);
}
-
+
private int memoizedSerializedSize = -1;
public int getSerializedSize() {
int size = memoizedSerializedSize;
if (size != -1) return size;
-
+
size = 0;
if (((bitField0_ & 0x00000001) == 0x00000001)) {
size += com.google.protobuf.CodedOutputStream
@@ -4306,14 +4363,14 @@ public final class ProtosFactory {
memoizedSerializedSize = size;
return size;
}
-
+
private static final long serialVersionUID = 0L;
@java.lang.Override
protected java.lang.Object writeReplace()
throws java.io.ObjectStreamException {
return super.writeReplace();
}
-
+
public static org.apache.flume.channel.file.proto.ProtosFactory.Commit parseFrom(
com.google.protobuf.ByteString data)
<TRUNCATED>
[3/3] git commit: FLUME-1586: File Channel should support verifying
integrity of individual events.
Posted by br...@apache.org.
FLUME-1586: File Channel should support verifying integrity of individual events.
(Hari Shreedharan via Brock Noland)
Project: http://git-wip-us.apache.org/repos/asf/flume/repo
Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/c57ebd1d
Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/c57ebd1d
Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/c57ebd1d
Branch: refs/heads/trunk
Commit: c57ebd1d2296fd1c0c8f84eecb16799de2a73e35
Parents: 791f443
Author: Brock Noland <br...@apache.org>
Authored: Wed Jun 12 17:35:38 2013 -0700
Committer: Brock Noland <br...@apache.org>
Committed: Wed Jun 12 17:35:38 2013 -0700
----------------------------------------------------------------------
bin/flume-ng | 6 +
.../channel/file/CorruptEventException.java | 36 +
.../apache/flume/channel/file/FileChannel.java | 47 +-
.../java/org/apache/flume/channel/file/Log.java | 12 +-
.../org/apache/flume/channel/file/LogFile.java | 104 +-
.../apache/flume/channel/file/LogFileV3.java | 18 +-
.../flume/channel/file/NoopRecordException.java | 35 +
.../java/org/apache/flume/channel/file/Put.java | 38 +-
.../flume/channel/file/ReplayHandler.java | 3 +-
.../flume/channel/file/Serialization.java | 15 +-
.../channel/file/TransactionEventRecord.java | 10 +-
.../flume/channel/file/proto/ProtosFactory.java | 1127 +++++++++---------
.../src/main/proto/filechannel.proto | 1 +
.../flume/channel/file/TestFileChannel.java | 44 +-
.../org/apache/flume/channel/file/TestLog.java | 28 +-
.../apache/flume/channel/file/TestLogFile.java | 111 +-
.../file/TestTransactionEventRecordV3.java | 16 +-
.../apache/flume/channel/file/TestUtils.java | 55 +-
flume-ng-dist/pom.xml | 4 +
flume-ng-dist/src/main/assembly/bin.xml | 1 +
flume-ng-dist/src/main/assembly/src.xml | 2 +
flume-tools/pom.xml | 143 +++
.../flume/tools/FileChannelIntegrityTool.java | 142 +++
.../java/org/apache/flume/tools/FlumeTool.java | 24 +
.../org/apache/flume/tools/FlumeToolType.java | 40 +
.../org/apache/flume/tools/FlumeToolsMain.java | 68 ++
.../tools/TestFileChannelIntegrityTool.java | 247 ++++
pom.xml | 7 +
28 files changed, 1765 insertions(+), 619 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/bin/flume-ng
----------------------------------------------------------------------
diff --git a/bin/flume-ng b/bin/flume-ng
index 22b95b8..65cc985 100755
--- a/bin/flume-ng
+++ b/bin/flume-ng
@@ -26,6 +26,7 @@
FLUME_AGENT_CLASS="org.apache.flume.node.Application"
FLUME_AVRO_CLIENT_CLASS="org.apache.flume.client.avro.AvroCLIClient"
FLUME_VERSION_CLASS="org.apache.flume.tools.VersionInfo"
+FLUME_TOOLS_CLASS="org.apache.flume.tools.FlumeToolsMain"
CLEAN_FLAG=1
################################
@@ -261,6 +262,9 @@ case "$mode" in
avro-client)
opt_avro_client=1
;;
+ tool)
+ opt_tool=1
+ ;;
version)
opt_version=1
CLEAN_FLAG=0
@@ -433,6 +437,8 @@ elif [ -n "$opt_avro_client" ] ; then
run_flume $FLUME_AVRO_CLIENT_CLASS $args
elif [ -n "${opt_version}" ] ; then
run_flume $FLUME_VERSION_CLASS $args
+elif [ -n "${opt_tool}" ] ; then
+ run_flume $FLUME_TOOLS_CLASS $args
else
error "This message should never appear" 1
fi
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
new file mode 100644
index 0000000..691d291
--- /dev/null
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/CorruptEventException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.channel.file;
+
+
+public class CorruptEventException extends Exception {
+
+ private static final long serialVersionUID = -2986946303540798416L;
+ public CorruptEventException() {
+ super();
+ }
+
+ public CorruptEventException(String msg) {
+ super(msg);
+ }
+
+ public CorruptEventException(String msg, Throwable th) {
+ super(msg, th);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
index cc0d38a..36f150b 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/FileChannel.java
@@ -100,6 +100,7 @@ public class FileChannel extends BasicChannelSemantics {
private String encryptionActiveKey;
private String encryptionCipherProvider;
private boolean useDualCheckpoints;
+ private boolean isTest = false;
@Override
public synchronized void setName(String name) {
@@ -451,6 +452,7 @@ public class FileChannel extends BasicChannelSemantics {
private String getStateAsString() {
return String.valueOf(getState());
}
+
@Override
protected void doPut(Event event) throws InterruptedException {
channelCounter.incrementEventPutAttemptCount();
@@ -511,23 +513,40 @@ public class FileChannel extends BasicChannelSemantics {
+ "log. Try increasing the log write timeout value. " +
channelNameDescriptor);
}
+
+ /*
+ * 1. Take an event which is in the queue.
+ * 2. If getting that event does not throw NoopRecordException,
+ * then return it.
+ * 3. Else try to retrieve the next event from the queue
+ * 4. Repeat 2 and 3 until queue is empty or an event is returned.
+ */
+
try {
- FlumeEventPointer ptr = queue.removeHead(transactionID);
- if(ptr != null) {
- try {
- // first add to takeList so that if write to disk
- // fails rollback actually does it's work
- Preconditions.checkState(takeList.offer(ptr), "takeList offer failed "
- + channelNameDescriptor);
- log.take(transactionID, ptr); // write take to disk
- Event event = log.get(ptr);
- return event;
- } catch (IOException e) {
- throw new ChannelException("Take failed due to IO error "
- + channelNameDescriptor, e);
+ while (true) {
+ FlumeEventPointer ptr = queue.removeHead(transactionID);
+ if (ptr == null) {
+ return null;
+ } else {
+ try {
+ // first add to takeList so that if write to disk
+ // fails rollback actually does it's work
+ Preconditions.checkState(takeList.offer(ptr),
+ "takeList offer failed "
+ + channelNameDescriptor);
+ log.take(transactionID, ptr); // write take to disk
+ Event event = log.get(ptr);
+ return event;
+ } catch (IOException e) {
+ throw new ChannelException("Take failed due to IO error "
+ + channelNameDescriptor, e);
+ } catch (NoopRecordException e) {
+ LOG.warn("Corrupt record replaced by File Channel Integrity " +
+ "tool found. Will retrieve next event", e);
+ takeList.remove(ptr);
+ }
}
}
- return null;
} finally {
log.unlockShared();
}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
index 1918baa..8dc0ff8 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
@@ -73,7 +73,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-class Log {
+public class Log {
public static final String PREFIX = "log-";
private static final Logger LOGGER = LoggerFactory.getLogger(Log.class);
private static final int MIN_NUM_LOGS = 2;
@@ -571,12 +571,18 @@ class Log {
* @throws InterruptedException
*/
FlumeEvent get(FlumeEventPointer pointer) throws IOException,
- InterruptedException {
+ InterruptedException, NoopRecordException {
Preconditions.checkState(open, "Log is closed");
int id = pointer.getFileID();
LogFile.RandomReader logFile = idLogFileMap.get(id);
Preconditions.checkNotNull(logFile, "LogFile is null for id " + id);
- return logFile.get(pointer.getOffset());
+ try {
+ return logFile.get(pointer.getOffset());
+ } catch (CorruptEventException ex) {
+ open = false;
+ throw new IOException("Corrupt event found. Please run File Channel " +
+ "Integrity tool.", ex);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
index d3db896..bb8ce1a 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
@@ -22,6 +22,9 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.flume.ChannelException;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.channel.file.encryption.CipherProvider;
import org.apache.flume.channel.file.encryption.KeyProvider;
import org.apache.flume.tools.DirectMemoryUtils;
@@ -31,6 +34,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.io.EOFException;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
@@ -40,7 +44,9 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
-abstract class LogFile {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class LogFile {
private static final Logger LOG = LoggerFactory
.getLogger(LogFile.class);
@@ -54,8 +60,9 @@ abstract class LogFile {
private static final ByteBuffer FILL = DirectMemoryUtils.
allocate(1024 * 1024); // preallocation, 1MB
- protected static final byte OP_RECORD = Byte.MAX_VALUE;
- protected static final byte OP_EOF = Byte.MIN_VALUE;
+ public static final byte OP_RECORD = Byte.MAX_VALUE;
+ public static final byte OP_NOOP = (Byte.MAX_VALUE + Byte.MIN_VALUE)/2;
+ public static final byte OP_EOF = Byte.MIN_VALUE;
static {
for (int i = 0; i < FILL.capacity(); i++) {
@@ -63,6 +70,13 @@ abstract class LogFile {
}
}
+ protected static void skipRecord(RandomAccessFile fileHandle,
+ int offset) throws IOException {
+ fileHandle.seek(offset);
+ int length = fileHandle.readInt();
+ fileHandle.skipBytes(length);
+ }
+
abstract static class MetaDataWriter {
private final File file;
private final int logFileID;
@@ -296,6 +310,48 @@ abstract class LogFile {
}
}
+ /**
+ * This is an class meant to be an internal Flume API,
+ * and can change at any time. Intended to be used only from File Channel Integrity
+ * test tool. Not to be used for any other purpose.
+ */
+ public static class OperationRecordUpdater {
+ private final RandomAccessFile fileHandle;
+ private final File file;
+
+ public OperationRecordUpdater(File file) throws FileNotFoundException {
+ Preconditions.checkState(file.exists(), "File to update, " +
+ file.toString() + " does not exist.");
+ this.file = file;
+ fileHandle = new RandomAccessFile(file, "rw");
+ }
+
+ public void markRecordAsNoop(long offset) throws IOException {
+ // First ensure that the offset actually is an OP_RECORD. There is a
+ // small possibility that it still is OP_RECORD,
+ // but is not actually the beginning of a record. Is there anything we
+ // can do about it?
+ fileHandle.seek(offset);
+ byte byteRead = fileHandle.readByte();
+ Preconditions.checkState(byteRead == OP_RECORD || byteRead == OP_NOOP,
+ "Expected to read a record but the byte read indicates EOF");
+ fileHandle.seek(offset);
+ LOG.info("Marking event as " + OP_NOOP + " at " + offset + " for file " +
+ file.toString());
+ fileHandle.writeByte(OP_NOOP);
+ }
+
+ public void close() {
+ try {
+ fileHandle.getFD().sync();
+ fileHandle.close();
+ } catch (IOException e) {
+ LOG.error("Could not close file handle to file " +
+ fileHandle.toString(), e);
+ }
+ }
+ }
+
static abstract class RandomReader {
private final File file;
private final BlockingQueue<RandomAccessFile> readFileHandles =
@@ -311,7 +367,7 @@ abstract class LogFile {
}
protected abstract TransactionEventRecord doGet(RandomAccessFile fileHandle)
- throws IOException;
+ throws IOException, CorruptEventException;
abstract int getVersion();
@@ -323,13 +379,18 @@ abstract class LogFile {
return encryptionKeyProvider;
}
- FlumeEvent get(int offset) throws IOException, InterruptedException {
+ FlumeEvent get(int offset) throws IOException, InterruptedException,
+ CorruptEventException, NoopRecordException {
Preconditions.checkState(open, "File closed");
RandomAccessFile fileHandle = checkOut();
boolean error = true;
try {
fileHandle.seek(offset);
byte operation = fileHandle.readByte();
+ if(operation == OP_NOOP) {
+ throw new NoopRecordException("No op record found. Corrupt record " +
+ "may have been repaired by File Channel Integrity tool");
+ }
Preconditions.checkState(operation == OP_RECORD,
Integer.toHexString(operation));
TransactionEventRecord record = doGet(fileHandle);
@@ -408,7 +469,7 @@ abstract class LogFile {
}
}
- static abstract class SequentialReader {
+ public static abstract class SequentialReader {
private final RandomAccessFile fileHandle;
private final FileChannel fileChannel;
@@ -434,7 +495,7 @@ abstract class LogFile {
fileHandle = new RandomAccessFile(file, "r");
fileChannel = fileHandle.getChannel();
}
- abstract LogRecord doNext(int offset) throws IOException;
+ abstract LogRecord doNext(int offset) throws IOException, CorruptEventException;
abstract int getVersion();
@@ -488,7 +549,7 @@ abstract class LogFile {
}
}
- LogRecord next() throws IOException {
+ public LogRecord next() throws IOException, CorruptEventException {
int offset = -1;
try {
long position = fileChannel.position();
@@ -499,14 +560,26 @@ abstract class LogFile {
}
offset = (int) position;
Preconditions.checkState(offset >= 0);
- byte operation = fileHandle.readByte();
- if(operation != OP_RECORD) {
- if(operation == OP_EOF) {
+ while (offset < fileHandle.length()) {
+ byte operation = fileHandle.readByte();
+ if (operation == OP_RECORD) {
+ break;
+ } else if (operation == OP_EOF) {
LOG.info("Encountered EOF at " + offset + " in " + file);
+ return null;
+ } else if (operation == OP_NOOP) {
+ LOG.info("No op event found in file: " + file.toString() +
+ " at " + offset + ". Skipping event.");
+ skipRecord(fileHandle, offset + 1);
+ offset = (int) fileHandle.getFilePointer();
+ continue;
} else {
LOG.error("Encountered non op-record at " + offset + " " +
- Integer.toHexString(operation) + " in " + file);
+ Integer.toHexString(operation) + " in " + file);
+ return null;
}
+ }
+ if(offset >= fileHandle.length()) {
return null;
}
return doNext(offset);
@@ -518,7 +591,10 @@ abstract class LogFile {
}
}
- void close() {
+ public long getPosition() throws IOException {
+ return fileChannel.position();
+ }
+ public void close() {
if(fileHandle != null) {
try {
fileHandle.close();
@@ -540,7 +616,7 @@ abstract class LogFile {
return buffer;
}
- public static void main(String[] args) throws EOFException, IOException {
+ public static void main(String[] args) throws EOFException, IOException, CorruptEventException {
File file = new File(args[0]);
LogFile.SequentialReader reader = null;
try {
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
index d9a2a9b..38f6ecb 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFileV3.java
@@ -21,6 +21,8 @@ package org.apache.flume.channel.file;
import com.google.common.base.Preconditions;
import com.google.protobuf.ByteString;
import com.google.protobuf.GeneratedMessage;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.channel.file.encryption.CipherProvider;
import org.apache.flume.channel.file.encryption.CipherProviderFactory;
import org.apache.flume.channel.file.encryption.KeyProvider;
@@ -43,7 +45,9 @@ import java.util.concurrent.LinkedBlockingDeque;
* Represents a single data file on disk. Has methods to write,
* read sequentially (replay), and read randomly (channel takes).
*/
-class LogFileV3 extends LogFile {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class LogFileV3 extends LogFile {
protected static final Logger LOGGER =
LoggerFactory.getLogger(LogFileV3.class);
@@ -267,7 +271,7 @@ class LogFileV3 extends LogFile {
}
@Override
protected TransactionEventRecord doGet(RandomAccessFile fileHandle)
- throws IOException {
+ throws IOException, CorruptEventException {
// readers are opened right when the file is created and thus
// empty. As such we wait to initialize until there is some
// data before we we initialize
@@ -297,10 +301,11 @@ class LogFileV3 extends LogFile {
}
}
- static class SequentialReader extends LogFile.SequentialReader {
+ public static class SequentialReader extends LogFile.SequentialReader {
private CipherProvider.Decryptor decryptor;
- SequentialReader(File file, @Nullable KeyProvider encryptionKeyProvider)
- throws EOFException, IOException {
+
+ public SequentialReader(File file, @Nullable KeyProvider
+ encryptionKeyProvider) throws EOFException, IOException {
super(file, encryptionKeyProvider);
File metaDataFile = Serialization.getMetaDataFile(file);
FileInputStream inputStream = new FileInputStream(metaDataFile);
@@ -344,8 +349,9 @@ class LogFileV3 extends LogFile {
public int getVersion() {
return Serialization.VERSION_3;
}
+
@Override
- LogRecord doNext(int offset) throws IOException {
+ LogRecord doNext(int offset) throws IOException, CorruptEventException {
byte[] buffer = readDelimitedBuffer(getFileHandle());
if(decryptor != null) {
buffer = decryptor.decrypt(buffer);
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
new file mode 100644
index 0000000..5f446b8
--- /dev/null
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/NoopRecordException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.channel.file;
+
+public class NoopRecordException extends Exception {
+ private static final long serialVersionUID = -7394180633208889738L;
+
+ public NoopRecordException() {
+ super();
+ }
+
+ public NoopRecordException(String msg) {
+ super(msg);
+ }
+
+ public NoopRecordException(String msg, Throwable th) {
+ super(msg, th);
+ }
+}
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
index 4235a79..f08f024 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Put.java
@@ -24,7 +24,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Map;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.flume.channel.file.proto.ProtosFactory;
import com.google.common.base.Preconditions;
@@ -36,13 +39,19 @@ import com.google.protobuf.ByteString;
*/
class Put extends TransactionEventRecord {
private FlumeEvent event;
+ // Should we move this to a higher level to not make multiple instances?
+ // Doing that might cause performance issues, since access to this would
+ // need to be synchronized (the whole reset-update-getValue cycle would
+ // need to be).
+ private final Checksum checksum = new CRC32();
+ @VisibleForTesting
Put(Long transactionID, Long logWriteOrderID) {
- super(transactionID, logWriteOrderID);
+ this(transactionID, logWriteOrderID, null);
}
Put(Long transactionID, Long logWriteOrderID, FlumeEvent event) {
- this(transactionID, logWriteOrderID);
+ super(transactionID, logWriteOrderID);
this.event = event;
}
@@ -78,11 +87,14 @@ class Put extends TransactionEventRecord {
}
}
eventBuilder.setBody(ByteString.copyFrom(event.getBody()));
- putBuilder.setEvent(eventBuilder.build());
+ ProtosFactory.FlumeEvent protoEvent = eventBuilder.build();
+ putBuilder.setEvent(protoEvent);
+ putBuilder.setChecksum(calculateChecksum(event.getBody()));
putBuilder.build().writeDelimitedTo(out);
}
@Override
- void readProtos(InputStream in) throws IOException {
+ void readProtos(InputStream in) throws IOException,
+ CorruptEventException {
ProtosFactory.Put put = Preconditions.checkNotNull(ProtosFactory.
Put.parseDelimitedFrom(in), "Put cannot be null");
Map<String, String> headers = Maps.newHashMap();
@@ -90,9 +102,25 @@ class Put extends TransactionEventRecord {
for(ProtosFactory.FlumeEventHeader header : protosEvent.getHeadersList()) {
headers.put(header.getKey(), header.getValue());
}
+ byte[] eventBody = protosEvent.getBody().toByteArray();
+
+ if (put.hasChecksum()) {
+ long eventBodyChecksum = calculateChecksum(eventBody);
+ if (eventBodyChecksum != put.getChecksum()) {
+ throw new CorruptEventException("Expected checksum for event was " +
+ eventBodyChecksum + " but the checksum of the event is " + put.getChecksum());
+ }
+ }
// TODO when we remove v2, remove FlumeEvent and use EventBuilder here
- event = new FlumeEvent(headers, protosEvent.getBody().toByteArray());
+ event = new FlumeEvent(headers, eventBody);
+ }
+
+ protected long calculateChecksum(byte[] body) {
+ checksum.reset();
+ checksum.update(body, 0, body.length);
+ return checksum.getValue();
}
+
@Override
public short getRecordType() {
return Type.PUT.get();
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
index fc47b23..c8f5fdd 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/ReplayHandler.java
@@ -26,6 +26,7 @@ import com.google.common.collect.SetMultimap;
import com.google.common.collect.Sets;
import org.apache.commons.collections.MultiMap;
import org.apache.commons.collections.map.MultiValueMap;
+import org.apache.flume.ChannelException;
import org.apache.flume.channel.file.encryption.KeyProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -373,7 +374,7 @@ class ReplayHandler {
}
}
}
- private LogRecord next() throws IOException {
+ private LogRecord next() throws IOException, CorruptEventException {
LogRecord resultLogRecord = logRecordBuffer.poll();
if(resultLogRecord != null) {
// there is more log records to read
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
index d6897e1..f8160d9 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Serialization.java
@@ -21,6 +21,8 @@ package org.apache.flume.channel.file;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import org.apache.commons.io.FileUtils;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,7 +35,9 @@ import java.io.RandomAccessFile;
import java.util.Collections;
import java.util.Set;
-class Serialization {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Serialization {
private Serialization() {}
static final long SIZE_OF_INT = 4;
@@ -43,9 +47,10 @@ class Serialization {
static final int VERSION_2 = 2;
static final int VERSION_3 = 3;
- static final String METADATA_FILENAME = ".meta";
- static final String METADATA_TMP_FILENAME = ".tmp";
- static final String OLD_METADATA_FILENAME = METADATA_FILENAME + ".old";
+ public static final String METADATA_FILENAME = ".meta";
+ public static final String METADATA_TMP_FILENAME = ".tmp";
+ public static final String OLD_METADATA_FILENAME = METADATA_FILENAME +
+ ".old";
// 64 K buffer to copy files.
private static final int FILE_COPY_BUFFER_SIZE = 64 * 1024;
@@ -121,7 +126,7 @@ class Serialization {
* @param to Destination file - this file should not exist
* @return true if the copy was successful
*/
- static boolean copyFile(File from, File to) throws IOException {
+ public static boolean copyFile(File from, File to) throws IOException {
Preconditions.checkNotNull(from, "Source file is null, file copy failed.");
Preconditions.checkNotNull(to, "Destination file is null, " +
"file copy failed.");
http://git-wip-us.apache.org/repos/asf/flume/blob/c57ebd1d/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
----------------------------------------------------------------------
diff --git a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
index 073042f..dda9b3f 100644
--- a/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
+++ b/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/TransactionEventRecord.java
@@ -29,6 +29,8 @@ import java.io.OutputStream;
import java.lang.reflect.Constructor;
import java.nio.ByteBuffer;
+import org.apache.flume.annotations.InterfaceAudience;
+import org.apache.flume.annotations.InterfaceStability;
import org.apache.flume.channel.file.proto.ProtosFactory;
import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
@@ -41,7 +43,9 @@ import com.google.common.collect.ImmutableMap;
/**
* Base class for records in data file: Put, Take, Rollback, Commit
*/
-abstract class TransactionEventRecord implements Writable {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class TransactionEventRecord implements Writable {
private static final Logger LOG = LoggerFactory
.getLogger(TransactionEventRecord.class);
private final long transactionID;
@@ -63,7 +67,7 @@ abstract class TransactionEventRecord implements Writable {
abstract void writeProtos(OutputStream out) throws IOException;
- abstract void readProtos(InputStream in) throws IOException;
+ abstract void readProtos(InputStream in) throws IOException, CorruptEventException;
long getLogWriteOrderID() {
return logWriteOrderID;
@@ -187,7 +191,7 @@ abstract class TransactionEventRecord implements Writable {
static TransactionEventRecord fromByteArray(byte[] buffer)
- throws IOException {
+ throws IOException, CorruptEventException {
ByteArrayInputStream in = new ByteArrayInputStream(buffer);
try {
ProtosFactory.TransactionEventHeader header = Preconditions.