You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by mp...@apache.org on 2012/06/26 14:35:14 UTC

svn commit: r1353995 - in /incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file: Log.java LogFile.java

Author: mpercy
Date: Tue Jun 26 12:35:13 2012
New Revision: 1353995

URL: http://svn.apache.org/viewvc?rev=1353995&view=rev
Log:
FLUME-1324. File Channel log can contain unassigned blocks.

(Arvind Prabhakar via Mike Percy)

Modified:
    incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
    incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java

Modified: incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java?rev=1353995&r1=1353994&r2=1353995&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java (original)
+++ incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java Tue Jun 26 12:35:13 2012
@@ -711,7 +711,9 @@ class Log {
           LogFile.RandomReader reader = idLogFileMap.remove(id);
           File file = reader.getFile();
           reader.close();
-          LogFile.Writer writer = new LogFile.Writer(file, id, maxFileSize);
+          // Open writer in inactive mode
+          LogFile.Writer writer =
+              new LogFile.Writer(file, id, maxFileSize, false);
           writer.markCheckpoint(ts);
           writer.close();
           reader = new LogFile.RandomReader(file);

Modified: incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java?rev=1353995&r1=1353994&r2=1353995&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java (original)
+++ incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java Tue Jun 26 12:35:13 2012
@@ -72,21 +72,51 @@ class LogFile {
 
     private volatile boolean open;
 
-    Writer(File file, int logFileID, long maxFileSize) throws IOException {
+    Writer(File file, int logFileID, long maxFileSize)
+        throws IOException {
+      this(file, logFileID, maxFileSize, true);
+    }
+
+    Writer(File file, int logFileID, long maxFileSize, boolean active)
+        throws IOException {
       this.file = file;
       fileID = logFileID;
       this.maxFileSize = Math.min(maxFileSize,
           FileChannelConfiguration.DEFAULT_MAX_FILE_SIZE);
       writeFileHandle = new RandomAccessFile(file, "rw");
-      writeFileHandle.writeInt(VERSION);
-      writeFileHandle.writeInt(fileID);
-      checkpointPositionMarker = writeFileHandle.getFilePointer();
-      // checkpoint marker
-      writeFileHandle.writeLong(0L);
-      // timestamp placeholder
-      writeFileHandle.writeLong(0L);
-      writeFileChannel = writeFileHandle.getChannel();
-      writeFileChannel.force(true);
+      if (active) {
+        writeFileHandle.writeInt(VERSION);
+        writeFileHandle.writeInt(fileID);
+        checkpointPositionMarker = writeFileHandle.getFilePointer();
+        // checkpoint marker
+        writeFileHandle.writeLong(0L);
+        // timestamp placeholder
+        writeFileHandle.writeLong(0L);
+        writeFileChannel = writeFileHandle.getChannel();
+        writeFileChannel.force(true);
+      } else {
+        int version = writeFileHandle.readInt();
+        if (version != VERSION) {
+          throw new IOException("The version of log file: "
+              + file.getCanonicalPath() + " is different from expected "
+              + " version: expected = " + VERSION + ", found = " + version);
+        }
+        int fid = writeFileHandle.readInt();
+        if (fid != logFileID) {
+          throw new IOException("The file id of log file: "
+              + file.getCanonicalPath() + " is different from expected "
+              + " id: expected = " + logFileID + ", found = " + fid);
+        }
+        checkpointPositionMarker = writeFileHandle.getFilePointer();
+        long chkptMarker = writeFileHandle.readLong();
+        long chkptTimestamp = writeFileHandle.readLong();
+        LOG.info("File: " + file.getCanonicalPath() + " was last checkpointed "
+             + "at position: " + chkptMarker + ", ts: " + chkptTimestamp);
+        writeFileChannel = writeFileHandle.getChannel();
+
+        // Jump to the last position
+        writeFileChannel.position(chkptMarker);
+      }
       LOG.info("Opened " + file);
       open = true;
     }
@@ -96,7 +126,7 @@ class LogFile {
     }
 
     synchronized void markCheckpoint(long timestamp) throws IOException {
-      long currentPosition = writeFileChannel.size();
+      long currentPosition = writeFileChannel.position();
       writeFileHandle.seek(checkpointPositionMarker);
       writeFileHandle.writeLong(currentPosition);
       writeFileHandle.writeLong(timestamp);
@@ -342,6 +372,7 @@ class LogFile {
         int offset = (int) position;
         byte operation = fileHandle.readByte();
         if(operation != OP_RECORD) {
+          LOG.info("Encountered non op-record at " + offset);
           return null;
         }
         TransactionEventRecord record = TransactionEventRecord.