You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2013/04/04 22:49:28 UTC
svn commit: r1464742 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hbase/regionserver/
Author: liyin
Date: Thu Apr 4 20:49:28 2013
New Revision: 1464742
URL: http://svn.apache.org/r1464742
Log:
[HBASE-5263] Letting the HFileWriter know, if a compaction is in progress or not
Author: rshroff
Summary:
We don't need to calculate whether the current block should be cached
because of compaction or not if the writer was not created by
compaction.
Also currently, its very difficult from the logs to know if the stats dumped
out for a Writer are for compaction or normal flush.
This change prevents calculating % of cached keys in case the writer is
not being used by compaction.
Test Plan: mr
Reviewers: liyintang, aaiyer, manukranthk
Reviewed By: manukranthk
CC: hbase-eng@
Differential Revision: https://phabricator.fb.com/D760745
Task ID: 2016630
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCachingOnCompaction.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java?rev=1464742&r1=1464741&r2=1464742&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java Thu Apr 4 20:49:28 2013
@@ -121,6 +121,12 @@ public abstract class AbstractHFileWrite
*/
protected int numKeysInCurrentBlock;
+ /**
+ * is compaction writer
+ */
+
+ protected boolean isCompactionWriter = false;
+
public AbstractHFileWriter(Configuration conf, CacheConfig cacheConf,
FSDataOutputStream outputStream, Path path, int blockSize,
Compression.Algorithm compressAlgo,
@@ -315,4 +321,8 @@ public abstract class AbstractHFileWrite
fs.getDefaultReplication(), fs.getDefaultBlockSize(), bytesPerChecksum,
null);
}
+
+ public void setCompactionWriter(boolean isCompaction) {
+ this.isCompactionWriter = isCompaction;
+ }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1464742&r1=1464741&r2=1464742&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Thu Apr 4 20:49:28 2013
@@ -262,6 +262,11 @@ public class HFile {
* @throws IOException
*/
void addDeleteColumnBloomFilter(BloomFilterWriter bfw) throws IOException;
+
+ /**
+ * Set whether compaction is in progress or not
+ */
+ void setCompactionWriter(boolean isCompaction);
}
/**
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1464742&r1=1464741&r2=1464742&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Thu Apr 4 20:49:28 2013
@@ -169,7 +169,7 @@ public class HFileWriterV2 extends Abstr
protected boolean cacheCurrentBlockForCompaction() {
- if (this.numKeysInCurrentBlock == 0) {
+ if (!this.isCompactionWriter || this.numKeysInCurrentBlock == 0) {
return false;
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1464742&r1=1464741&r2=1464742&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Apr 4 20:49:28 2013
@@ -797,6 +797,7 @@ public class Store extends SchemaConfigu
.withFavoredNodes(region.getFavoredNodes())
.withCompression(compression)
.build();
+ w.getHFileWriter().setCompactionWriter(isCompaction);
// The store file writer's path does not include the CF name, so we need
// to configure the HFile writer directly.
SchemaConfigured sc = (SchemaConfigured) w.writer;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCachingOnCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCachingOnCompaction.java?rev=1464742&r1=1464741&r2=1464742&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCachingOnCompaction.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCachingOnCompaction.java Thu Apr 4 20:49:28 2013
@@ -42,7 +42,7 @@ import org.junit.Test;
* Test caching on Compaction
*/
public class TestCachingOnCompaction {
- static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
+ static final Log LOG = LogFactory.getLog(TestCachingOnCompaction.class.getName());
private int compactionThreshold;
byte [] TABLE = null;
HTable ht = null;