You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2013/03/15 02:41:45 UTC
svn commit: r1456743 - in
/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase:
io/hfile/AbstractHFileWriter.java io/hfile/HFile.java
regionserver/StoreFile.java
Author: sershe
Date: Fri Mar 15 01:41:45 2013
New Revision: 1456743
URL: http://svn.apache.org/r1456743
Log:
HBASE-8034 record on-disk data size for store file and make it available during writing
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java?rev=1456743&r1=1456742&r2=1456743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java Fri Mar 15 01:41:45 2013
@@ -266,4 +266,10 @@ public abstract class AbstractHFileWrite
HConstants.DATA_FILE_UMASK_KEY);
return FSUtils.create(fs, path, perms);
}
+
+ @Override
+ public long getCurrentSize() throws IOException {
+ if (this.outputStream == null) return -1;
+ return this.outputStream.getPos();
+ }
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1456743&r1=1456742&r2=1456743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Fri Mar 15 01:41:45 2013
@@ -319,6 +319,11 @@ public class HFile {
* HFile V2.
*/
void addDeleteFamilyBloomFilter(BloomFilterWriter bfw) throws IOException;
+
+ /**
+ * @return Currently written raw data size on disk.
+ */
+ long getCurrentSize() throws IOException;
}
/**
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1456743&r1=1456742&r2=1456743&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Fri Mar 15 01:41:45 2013
@@ -107,6 +107,9 @@ public class StoreFile {
public static final byte[] DELETE_FAMILY_COUNT =
Bytes.toBytes("DELETE_FAMILY_COUNT");
+ /** See {@link #getEstimatedDiskDataSize()}. */
+ public static final byte[] DISK_DATA_SIZE_KEY = Bytes.toBytes("DISK_DATA_SIZE");
+
/** Last Bloom filter key in FileInfo */
private static final byte[] LAST_BLOOM_KEY = Bytes.toBytes("LAST_BLOOM_KEY");
@@ -149,6 +152,12 @@ public class StoreFile {
// whenever you get a Reader.
private AtomicBoolean majorCompaction = null;
+ /** See {@link #getEstimatedDiskDataSize()}. */
+ private long diskDataSize;
+
+ /** See {@link #getEstimatedDiskDataSize()}. */
+ private static double DATA_SIZE_FRACTION_ESTIMATE = 0.98;
+
// If true, this file should not be included in minor compactions.
// It's set whenever you get a Reader.
private boolean excludeFromMinorCompaction = false;
@@ -288,6 +297,15 @@ public class StoreFile {
}
/**
+ * @return Estimated number of bytes taken by the data blocks of this file. Either the exact
+ * number written into the file metadata ({@link #DISK_DATA_SIZE_KEY}); or estimated as
+ * {@link #DATA_SIZE_FRACTION_ESTIMATE} of the file, if there's no such field (old files).
+ */
+ public long getEstimatedDiskDataSize() {
+ return diskDataSize;
+ }
+
+ /**
* Return the largest memstoreTS found across all storefiles in
* the given list. Store files that were created by a mapreduce
* bulk load are ignored, as they do not correspond to any specific
@@ -446,6 +464,12 @@ public class StoreFile {
"proceeding without", e);
this.reader.timeRangeTracker = null;
}
+
+ b = metadataMap.get(DISK_DATA_SIZE_KEY);
+ // Estimate which fraction of the file is data if the file doesn't have this field.
+ this.diskDataSize = (b != null)
+ ? Bytes.toLong(b) : (long)(this.reader.length() * DATA_SIZE_FRACTION_ESTIMATE);
+
return this.reader;
}
@@ -1073,6 +1097,12 @@ public class StoreFile {
}
public void close() throws IOException {
+ // Estimate data size in this file before blooms and the HFile tail blocks.
+ long currentSize = writer.getCurrentSize();
+ if (currentSize >= 0) {
+ writer.appendFileInfo(DISK_DATA_SIZE_KEY, Bytes.toBytes(currentSize));
+ }
+
boolean hasGeneralBloom = this.closeGeneralBloomFilter();
boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
@@ -1095,6 +1125,10 @@ public class StoreFile {
HFile.Writer getHFileWriter() {
return writer;
}
+
+ public long getCurrentSize() throws IOException {
+ return writer.getCurrentSize();
+ }
}
/**