You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:16:33 UTC
svn commit: r1181523 -
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
Author: nspiegelberg
Date: Tue Oct 11 02:16:32 2011
New Revision: 1181523
URL: http://svn.apache.org/viewvc?rev=1181523&view=rev
Log:
SequenceFileLogWriter can create forceSync log file based on conf var
Summary:
1) HDFS-744: HDFS sync on sync can make sure all the sync will write the data into disk
2) HDFS: change the file system interface in hadoop core package will expose the new create interface
So this change will use the NEW create api based on the conf var.
To be backward compatible, the default value should be false and it still call
the OLD create api in HDFS. Only call the new create api when this value is
true.
Test Plan:
Test Locally
Reviewed By: kannan
Reviewers: hkuang, kannan, nspiegelberg, kranganathan, dhruba, jgray
Commenters: dhruba
CC: hbase@lists, , hbase-hdfs@lists, dhruba, kannan
Differential Revision: 235189
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java?rev=1181523&r1=1181522&r2=1181523&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogWriter.java Tue Oct 11 02:16:32 2011
@@ -57,28 +57,10 @@ public class SequenceFileLogWriter imple
@Override
public void init(FileSystem fs, Path path, Configuration conf)
- throws IOException {
+ throws IOException {
// Create a SF.Writer instance.
try {
- // reflection for a version of SequenceFile.createWriter that doesn't
- // automatically create the parent directory (see HBASE-2312)
- this.writer = (SequenceFile.Writer) SequenceFile.class
- .getMethod("createWriter", new Class[] {FileSystem.class,
- Configuration.class, Path.class, Class.class, Class.class,
- Integer.TYPE, Short.TYPE, Long.TYPE, Boolean.TYPE,
- CompressionType.class, CompressionCodec.class, Metadata.class})
- .invoke(null, new Object[] {fs, conf, path, HLog.getKeyClass(conf),
- WALEdit.class,
- new Integer(fs.getConf().getInt("io.file.buffer.size", 4096)),
- new Short((short)
- conf.getInt("hbase.regionserver.hlog.replication",
- fs.getDefaultReplication())),
- new Long(conf.getLong("hbase.regionserver.hlog.blocksize",
- fs.getDefaultBlockSize())),
- new Boolean(false) /*createParent*/,
- SequenceFile.CompressionType.NONE, new DefaultCodec(),
- new Metadata()
- });
+ this.generateWriter(fs,path,conf);
} catch (InvocationTargetException ite) {
// function was properly called, but threw it's own exception
throw new IOException(ite.getCause());
@@ -86,8 +68,6 @@ public class SequenceFileLogWriter imple
// ignore all other exceptions. related to reflection failure
}
-
-
// if reflection failed, use the old createWriter
if (this.writer == null) {
LOG.warn("new createWriter -- HADOOP-6840 -- not available");
@@ -178,4 +158,56 @@ public class SequenceFileLogWriter imple
public OutputStream getDFSCOutputStream() {
return this.dfsClient_out;
}
+
+ // To be backward compatible; we still need to call the old sequence file
+ // interface.
+ private void generateWriter(FileSystem fs, Path path, Configuration conf)
+ throws InvocationTargetException, Exception {
+ boolean forceSync =
+ conf.getBoolean("hbase.regionserver.hlog.writer.forceSync", false);
+ if (forceSync) {
+ // call the new create api with force sync flag
+ this.writer = (SequenceFile.Writer) SequenceFile.class
+ .getMethod("createWriter", new Class[] {FileSystem.class,
+ Configuration.class, Path.class, Class.class, Class.class,
+ Integer.TYPE, Short.TYPE, Long.TYPE, Boolean.TYPE,
+ CompressionType.class, CompressionCodec.class, Metadata.class,
+ Boolean.TYPE})
+ .invoke(null, new Object[] {fs, conf, path, HLog.getKeyClass(conf),
+ WALEdit.class,
+ new Integer(fs.getConf().getInt("io.file.buffer.size", 4096)),
+ new Short((short)
+ conf.getInt("hbase.regionserver.hlog.replication",
+ fs.getDefaultReplication())),
+ new Long(conf.getLong("hbase.regionserver.hlog.blocksize",
+ fs.getDefaultBlockSize())),
+ new Boolean(false) /*createParent*/,
+ SequenceFile.CompressionType.NONE, new DefaultCodec(),
+ new Metadata(),
+ forceSync
+ });
+
+ } else {
+ // still need to keep old interface to be backward compatible
+ // reflection for a version of SequenceFile.createWriter that doesn't
+ // automatically create the parent directory (see HBASE-2312)
+ this.writer = (SequenceFile.Writer) SequenceFile.class
+ .getMethod("createWriter", new Class[] {FileSystem.class,
+ Configuration.class, Path.class, Class.class, Class.class,
+ Integer.TYPE, Short.TYPE, Long.TYPE, Boolean.TYPE,
+ CompressionType.class, CompressionCodec.class, Metadata.class})
+ .invoke(null, new Object[] {fs, conf, path, HLog.getKeyClass(conf),
+ WALEdit.class,
+ new Integer(fs.getConf().getInt("io.file.buffer.size", 4096)),
+ new Short((short)
+ conf.getInt("hbase.regionserver.hlog.replication",
+ fs.getDefaultReplication())),
+ new Long(conf.getLong("hbase.regionserver.hlog.blocksize",
+ fs.getDefaultBlockSize())),
+ new Boolean(false) /*createParent*/,
+ SequenceFile.CompressionType.NONE, new DefaultCodec(),
+ new Metadata()
+ });
+ }
+ }
}