You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by mp...@apache.org on 2012/06/26 14:35:14 UTC
svn commit: r1353995 - in
/incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file:
Log.java LogFile.java
Author: mpercy
Date: Tue Jun 26 12:35:13 2012
New Revision: 1353995
URL: http://svn.apache.org/viewvc?rev=1353995&view=rev
Log:
FLUME-1324. File Channel log can contain unassigned blocks.
(Arvind Prabhakar via Mike Percy)
Modified:
incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
Modified: incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java?rev=1353995&r1=1353994&r2=1353995&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java (original)
+++ incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/Log.java Tue Jun 26 12:35:13 2012
@@ -711,7 +711,9 @@ class Log {
LogFile.RandomReader reader = idLogFileMap.remove(id);
File file = reader.getFile();
reader.close();
- LogFile.Writer writer = new LogFile.Writer(file, id, maxFileSize);
+ // Open writer in inactive mode
+ LogFile.Writer writer =
+ new LogFile.Writer(file, id, maxFileSize, false);
writer.markCheckpoint(ts);
writer.close();
reader = new LogFile.RandomReader(file);
Modified: incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java?rev=1353995&r1=1353994&r2=1353995&view=diff
==============================================================================
--- incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java (original)
+++ incubator/flume/trunk/flume-ng-channels/flume-file-channel/src/main/java/org/apache/flume/channel/file/LogFile.java Tue Jun 26 12:35:13 2012
@@ -72,21 +72,51 @@ class LogFile {
private volatile boolean open;
- Writer(File file, int logFileID, long maxFileSize) throws IOException {
+ Writer(File file, int logFileID, long maxFileSize)
+ throws IOException {
+ this(file, logFileID, maxFileSize, true);
+ }
+
+ Writer(File file, int logFileID, long maxFileSize, boolean active)
+ throws IOException {
this.file = file;
fileID = logFileID;
this.maxFileSize = Math.min(maxFileSize,
FileChannelConfiguration.DEFAULT_MAX_FILE_SIZE);
writeFileHandle = new RandomAccessFile(file, "rw");
- writeFileHandle.writeInt(VERSION);
- writeFileHandle.writeInt(fileID);
- checkpointPositionMarker = writeFileHandle.getFilePointer();
- // checkpoint marker
- writeFileHandle.writeLong(0L);
- // timestamp placeholder
- writeFileHandle.writeLong(0L);
- writeFileChannel = writeFileHandle.getChannel();
- writeFileChannel.force(true);
+ if (active) {
+ writeFileHandle.writeInt(VERSION);
+ writeFileHandle.writeInt(fileID);
+ checkpointPositionMarker = writeFileHandle.getFilePointer();
+ // checkpoint marker
+ writeFileHandle.writeLong(0L);
+ // timestamp placeholder
+ writeFileHandle.writeLong(0L);
+ writeFileChannel = writeFileHandle.getChannel();
+ writeFileChannel.force(true);
+ } else {
+ int version = writeFileHandle.readInt();
+ if (version != VERSION) {
+ throw new IOException("The version of log file: "
+ + file.getCanonicalPath() + " is different from expected "
+ + " version: expected = " + VERSION + ", found = " + version);
+ }
+ int fid = writeFileHandle.readInt();
+ if (fid != logFileID) {
+ throw new IOException("The file id of log file: "
+ + file.getCanonicalPath() + " is different from expected "
+ + " id: expected = " + logFileID + ", found = " + fid);
+ }
+ checkpointPositionMarker = writeFileHandle.getFilePointer();
+ long chkptMarker = writeFileHandle.readLong();
+ long chkptTimestamp = writeFileHandle.readLong();
+ LOG.info("File: " + file.getCanonicalPath() + " was last checkpointed "
+ + "at position: " + chkptMarker + ", ts: " + chkptTimestamp);
+ writeFileChannel = writeFileHandle.getChannel();
+
+ // Jump to the last position
+ writeFileChannel.position(chkptMarker);
+ }
LOG.info("Opened " + file);
open = true;
}
@@ -96,7 +126,7 @@ class LogFile {
}
synchronized void markCheckpoint(long timestamp) throws IOException {
- long currentPosition = writeFileChannel.size();
+ long currentPosition = writeFileChannel.position();
writeFileHandle.seek(checkpointPositionMarker);
writeFileHandle.writeLong(currentPosition);
writeFileHandle.writeLong(timestamp);
@@ -342,6 +372,7 @@ class LogFile {
int offset = (int) position;
byte operation = fileHandle.readByte();
if(operation != OP_RECORD) {
+ LOG.info("Encountered non op-record at " + offset);
return null;
}
TransactionEventRecord record = TransactionEventRecord.