You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/06/16 23:18:09 UTC
svn commit: r1136686 - in /hbase/trunk: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/io/hfile/
src/main/java/org/apache/hadoop/hbase/regionserver/
src/main/java/org/apache...
Author: stack
Date: Thu Jun 16 21:18:08 2011
New Revision: 1136686
URL: http://svn.apache.org/viewvc?rev=1136686&view=rev
Log:
HBASE-3927 display total uncompressed byte size of a region in web UI
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Jun 16 21:18:08 2011
@@ -266,6 +266,7 @@ Release 0.91.0 - Unreleased
HBASE-3940 HBase daemons should log version info at startup and possibly
periodically (Li Pi)
HBASE-3789 Cleanup the locking contention in the master
+ HBASE-3927 Display total uncompressed byte size of a region in web UI
TASKS
HBASE-3559 Move report of split to master OFF the heartbeat channel
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HServerLoad.java Thu Jun 16 21:18:08 2011
@@ -36,7 +36,7 @@ import org.apache.hadoop.io.WritableComp
*/
public class HServerLoad extends VersionedWritable
implements WritableComparable<HServerLoad> {
- private static final byte VERSION = 1;
+ private static final byte VERSION = 2;
// Empty load instance.
public static final HServerLoad EMPTY_HSERVERLOAD = new HServerLoad();
@@ -64,7 +64,7 @@ implements WritableComparable<HServerLoa
* Encapsulates per-region loading metrics.
*/
public static class RegionLoad extends VersionedWritable {
- private static final byte VERSION = 0;
+ private static final byte VERSION = 1;
/** @return the object version number */
public byte getVersion() {
@@ -77,6 +77,8 @@ implements WritableComparable<HServerLoa
private int stores;
/** the number of storefiles for the region */
private int storefiles;
+ /** the total size of the store files for the region, uncompressed, in MB */
+ private int storeUncompressedSizeMB;
/** the current total size of the store files for the region, in MB */
private int storefileSizeMB;
/** the current size of the memstore for the region, in MB */
@@ -106,12 +108,14 @@ implements WritableComparable<HServerLoa
* @param writeRequestsCount
*/
public RegionLoad(final byte[] name, final int stores,
- final int storefiles, final int storefileSizeMB,
+ final int storefiles, final int storeUncompressedSizeMB,
+ final int storefileSizeMB,
final int memstoreSizeMB, final int storefileIndexSizeMB,
final int readRequestsCount, final int writeRequestsCount) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
+ this.storeUncompressedSizeMB = storeUncompressedSizeMB;
this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
@@ -246,13 +250,14 @@ implements WritableComparable<HServerLoa
// Writable
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- int version = getVersion();
- if (version != VERSION) throw new IOException("Version mismatch; " + version);
+ int version = in.readByte();
+ if (version > VERSION) throw new IOException("Version mismatch; " + version);
int namelen = in.readInt();
this.name = new byte[namelen];
in.readFully(this.name);
this.stores = in.readInt();
this.storefiles = in.readInt();
+ this.storeUncompressedSizeMB = in.readInt();
this.storefileSizeMB = in.readInt();
this.memstoreSizeMB = in.readInt();
this.storefileIndexSizeMB = in.readInt();
@@ -262,10 +267,12 @@ implements WritableComparable<HServerLoa
public void write(DataOutput out) throws IOException {
super.write(out);
+ out.writeByte(VERSION);
out.writeInt(name.length);
out.write(name);
out.writeInt(stores);
out.writeInt(storefiles);
+ out.writeInt(storeUncompressedSizeMB);
out.writeInt(storefileSizeMB);
out.writeInt(memstoreSizeMB);
out.writeInt(storefileIndexSizeMB);
@@ -282,8 +289,15 @@ implements WritableComparable<HServerLoa
Integer.valueOf(this.stores));
sb = Strings.appendKeyValue(sb, "storefiles",
Integer.valueOf(this.storefiles));
+ sb = Strings.appendKeyValue(sb, "storefileUncompressedSizeMB",
+ Integer.valueOf(this.storeUncompressedSizeMB));
sb = Strings.appendKeyValue(sb, "storefileSizeMB",
Integer.valueOf(this.storefileSizeMB));
+ if (this.storeUncompressedSizeMB != 0) {
+ sb = Strings.appendKeyValue(sb, "compressionRatio",
+ String.format("%.4f", (float)this.storefileSizeMB/
+ (float)this.storeUncompressedSizeMB));
+ }
sb = Strings.appendKeyValue(sb, "memstoreSizeMB",
Integer.valueOf(this.memstoreSizeMB));
sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
@@ -480,8 +494,8 @@ implements WritableComparable<HServerLoa
public void readFields(DataInput in) throws IOException {
super.readFields(in);
- int version = getVersion();
- if (version != VERSION) throw new IOException("Version mismatch; " + version);
+ int version = in.readByte();
+ if (version > VERSION) throw new IOException("Version mismatch; " + version);
numberOfRequests = in.readInt();
usedHeapMB = in.readInt();
maxHeapMB = in.readInt();
@@ -495,6 +509,7 @@ implements WritableComparable<HServerLoa
public void write(DataOutput out) throws IOException {
super.write(out);
+ out.writeByte(VERSION);
out.writeInt(numberOfRequests);
out.writeInt(usedHeapMB);
out.writeInt(maxHeapMB);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Thu Jun 16 21:18:08 2011
@@ -21,10 +21,8 @@ package org.apache.hadoop.hbase.client;
import java.io.Closeable;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
-import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Thu Jun 16 21:18:08 2011
@@ -841,6 +841,10 @@ public class HFile {
return this.fileSize;
}
+ public long getTotalUncompressedBytes() {
+ return this.trailer.totalUncompressedBytes;
+ }
+
public boolean inMemory() {
return this.inMemory;
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Jun 16 21:18:08 2011
@@ -908,6 +908,7 @@ public class HRegionServer implements HR
byte[] name = r.getRegionName();
int stores = 0;
int storefiles = 0;
+ int storeUncompressedSizeMB = 0;
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.memstoreSize.get() / 1024 / 1024);
int storefileIndexSizeMB = 0;
@@ -915,11 +916,14 @@ public class HRegionServer implements HR
stores += r.stores.size();
for (Store store : r.stores.values()) {
storefiles += store.getStorefilesCount();
+ storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed()
+ / 1024 / 1024);
storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
storefileIndexSizeMB += (int) (store.getStorefilesIndexSize() / 1024 / 1024);
}
}
return new HServerLoad.RegionLoad(name,stores, storefiles,
+ storeUncompressedSizeMB,
storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB,
(int) r.readRequestsCount.get(), (int) r.writeRequestsCount.get());
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Jun 16 21:18:08 2011
@@ -110,6 +110,7 @@ public class Store implements HeapSize {
private final long desiredMaxFileSize;
private final int blockingStoreFileCount;
private volatile long storeSize = 0L;
+ private volatile long totalUncompressedBytes = 0L;
private final Object flushLock = new Object();
final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final String storeNameStr;
@@ -288,6 +289,7 @@ public class Store implements HeapSize {
}
long length = curfile.getReader().length();
this.storeSize += length;
+ this.totalUncompressedBytes += curfile.getReader().getTotalUncompressedBytes();
if (LOG.isDebugEnabled()) {
LOG.debug("loaded " + curfile.toStringDetailed());
}
@@ -525,6 +527,7 @@ public class Store implements HeapSize {
this.conf, this.family.getBloomFilterType(), this.inMemory);
StoreFile.Reader r = sf.createReader();
this.storeSize += r.length();
+ this.totalUncompressedBytes += r.getTotalUncompressedBytes();
if(LOG.isInfoEnabled()) {
LOG.info("Added " + sf + ", entries=" + r.getEntries() +
", sequenceid=" + logCacheFlushId +
@@ -1208,6 +1211,7 @@ public class Store implements HeapSize {
}
// 4. Compute new store size
this.storeSize = 0L;
+ this.totalUncompressedBytes = 0L;
for (StoreFile hsf : this.storefiles) {
StoreFile.Reader r = hsf.getReader();
if (r == null) {
@@ -1215,6 +1219,7 @@ public class Store implements HeapSize {
continue;
}
this.storeSize += r.length();
+ this.totalUncompressedBytes += r.getTotalUncompressedBytes();
}
} finally {
this.lock.writeLock().unlock();
@@ -1531,6 +1536,13 @@ public class Store implements HeapSize {
}
/**
+ * @return The size of the store files, in bytes, uncompressed.
+ */
+ long getStoreSizeUncompressed() {
+ return this.totalUncompressedBytes;
+ }
+
+ /**
* @return The size of the store files, in bytes.
*/
long getStorefilesSize() {
@@ -1684,7 +1696,7 @@ public class Store implements HeapSize {
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT + (15 * ClassSize.REFERENCE) +
- (7 * Bytes.SIZEOF_LONG) + (1 * Bytes.SIZEOF_DOUBLE) +
+ (8 * Bytes.SIZEOF_LONG) + (1 * Bytes.SIZEOF_DOUBLE) +
(4 * Bytes.SIZEOF_INT) + (3 * Bytes.SIZEOF_BOOLEAN));
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD +
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Thu Jun 16 21:18:08 2011
@@ -1092,6 +1092,10 @@ public class StoreFile {
return reader.length();
}
+ public long getTotalUncompressedBytes() {
+ return reader.getTotalUncompressedBytes();
+ }
+
public int getEntries() {
return reader.getEntries();
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java?rev=1136686&r1=1136685&r2=1136686&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java Thu Jun 16 21:18:08 2011
@@ -353,7 +353,7 @@ public class HLogSplitter {
}
public boolean splitLogFileToTemp(FileStatus logfile, String tmpname,
- CancelableProgressable reporter) throws IOException {
+ CancelableProgressable reporter) throws IOException {
final Map<byte[], Object> logWriters = Collections.
synchronizedMap(new TreeMap<byte[], Object>(Bytes.BYTES_COMPARATOR));
boolean isCorrupted = false;
@@ -409,7 +409,10 @@ public class HLogSplitter {
if (wap == null) {
wap = createWAP(region, entry, rootDir, tmpname, fs, conf);
if (wap == null) {
+ // ignore edits from this region. It doesn't ezist anymore.
+ // It was probably already split.
logWriters.put(region, BAD_WRITER);
+ continue;
} else {
logWriters.put(region, wap);
}