You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/07/17 01:55:25 UTC
svn commit: r794910 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/ipc/
src/java/org/apache/hadoop/hbase/regionserver/
Author: apurtell
Date: Thu Jul 16 23:55:25 2009
New Revision: 794910
URL: http://svn.apache.org/viewvc?rev=794910&view=rev
Log:
HBASE-1665
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=794910&r1=794909&r2=794910&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Thu Jul 16 23:55:25 2009
@@ -102,8 +102,8 @@
HBASE-1292 php thrift's getRow() would throw an exception if the row does
not exist (Rong-en Fan via Stack)
HBASE-1340 Fix new javadoc warnings (Evgeny Ryabitskiy via Stack)
- HBASE-1287 Partitioner class not used in TableMapReduceUtil.initTableReduceJob()
- (Lars George and Billy Pearson via Stack)
+ HBASE-1287 Partitioner class not used in TableMapReduceUtil
+ .initTableReduceJob() (Lars George and Billy Pearson via Stack)
HBASE-1320 hbase-1234 broke filter tests
HBASE-1355 [performance] Cache family maxversions; we were calculating on
each access
@@ -238,16 +238,17 @@
HBASE-1437 broken links in hbase.org
HBASE-1582 Translate ColumnValueFilter and RowFilterSet to the new Filter
interface
- HBASE-1594 Fix scan addcolumns after hbase-1385 commit (broken hudson build)
+ HBASE-1594 Fix scan addcolumns after hbase-1385 commit (broke hudson build)
HBASE-1595 hadoop-default.xml and zoo.cfg in hbase jar
HBASE-1602 HRegionServer won't go down since we added in new LruBlockCache
HBASE-1608 TestCachedBlockQueue failing on some jvms (Jon Gray via Stack)
HBASE-1615 HBASE-1597 introduced a bug when compacting after a split
(Jon Gray via Stack)
- HBASE-1616 Unit test of compacting referenced StoreFiles (Jon Gray via Stack)
+ HBASE-1616 Unit test of compacting referenced StoreFiles (Jon Gray via
+ Stack)
HBASE-1618 Investigate further into the MemStoreFlusher StoreFile limit
(Jon Gray via Stack)
- HBASE-1625 Adding check to Put.add(KeyValue kv), to see that it has the same
+ HBASE-1625 Adding check to Put.add(KeyValue), to see that it has the same
row as when instantiated (Erik Holstad via Stack)
HBASE-1629 HRS unable to contact master
HBASE-1633 Can't delete in TRUNK shell; makes it hard doing admin repairs
@@ -270,6 +271,7 @@
HBASE-1663 Request compaction only once instead of every time 500ms each
time we cycle the hstore.getStorefilesCount() >
this.blockingStoreFilesNumber loop
+ HBASE-1058 Disable 1058 on catalog tables
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
@@ -452,7 +454,8 @@
(Erik Holstad via Stack)
HBASE-1584 Put add methods should return this for ease of use (Be
consistant with Get) (Clint Morgan via Stack)
- HBASE-1581 Run major compaction on .META. when table is dropped or truncated
+ HBASE-1581 Run major compaction on .META. when table is dropped or
+ truncated
HBASE-1587 Update ganglia config and doc to account for ganglia 3.1 and
hadoop-4675
HBASE-1589 Up zk maxClientCnxns from default of 10 to 20 or 30 or so
@@ -468,23 +471,25 @@
HBASE-1218 Implement in-memory column (Jon Gray via Stack)
HBASE-1606 Remove zoo.cfg, put config options into hbase-site.xml
HBASE-1575 HMaster does not handle ZK session expiration
- HBASE-1620 Need to use special StoreScanner constructor for major compactions
- (passed sf, no caching, etc) (Jon Gray via Stack)
+ HBASE-1620 Need to use special StoreScanner constructor for major
+ compactions (passed sf, no caching, etc) (Jon Gray via Stack)
HBASE-1624 Don't sort Puts if only one in list in HCM#processBatchOfRows
HBASE-1626 Allow emitting Deletes out of new TableReducer
(Lars George via Stack)
HBASE-1551 HBase should manage multiple node ZooKeeper quorum
HBASE-1637 Delete client class methods should return itself like Put, Get,
Scan (Jon Gray via Nitay)
- HBASE-1640 Allow passing arguments to jruby script run when run by bin/hbase shell
+ HBASE-1640 Allow passing arguments to jruby script run when run by hbase
+ shell
HBASE-698 HLog recovery is not performed after master failure
HBASE-1643 ScanDeleteTracker takes comparator but it unused
HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region
server Some server for region TestTable..." -- deubugging
HBASE-1470 hbase and HADOOP-4379, dhruba's flush/sync
- HBASE-1632 Write documentation for configuring/managing ZooKeeper with HBase
+ HBASE-1632 Write documentation for configuring/managing ZooKeeper
HBASE-1662 Tool to run major compaction on catalog regions when hbase is
shutdown
+ HBASE-1665 expose more load information to the client side
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java?rev=794910&r1=794909&r2=794910&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/HServerLoad.java Thu Jul 16 23:55:25 2009
@@ -59,6 +59,8 @@
private int stores;
/** the number of storefiles for the region */
private int storefiles;
+ /** the current total size of the store files for the region, in MB */
+ private int storefileSizeMB;
/** the current size of the memstore for the region, in MB */
private int memstoreSizeMB;
/** the current total size of storefile indexes for the region, in MB */
@@ -75,15 +77,17 @@
* @param name
* @param stores
* @param storefiles
+ * @param storefileSizeMB
* @param memstoreSizeMB
* @param storefileIndexSizeMB
*/
public RegionLoad(final byte[] name, final int stores,
- final int storefiles, final int memstoreSizeMB,
- final int storefileIndexSizeMB) {
+ final int storefiles, final int storefileSizeMB,
+ final int memstoreSizeMB, final int storefileIndexSizeMB) {
this.name = name;
this.stores = stores;
this.storefiles = storefiles;
+ this.storefileSizeMB = storefileSizeMB;
this.memstoreSizeMB = memstoreSizeMB;
this.storefileIndexSizeMB = storefileIndexSizeMB;
}
@@ -119,6 +123,13 @@
}
/**
+ * @return the total size of the storefiles, in MB
+ */
+ public int getStorefileSizeMB() {
+ return storefileSizeMB;
+ }
+
+ /**
* @return the memstore size, in MB
*/
public int getMemStoreSizeMB() {
@@ -177,6 +188,7 @@
in.readFully(this.name);
this.stores = in.readInt();
this.storefiles = in.readInt();
+ this.storefileSizeMB = in.readInt();
this.memstoreSizeMB = in.readInt();
this.storefileIndexSizeMB = in.readInt();
}
@@ -186,6 +198,7 @@
out.write(name);
out.writeInt(stores);
out.writeInt(storefiles);
+ out.writeInt(storefileSizeMB);
out.writeInt(memstoreSizeMB);
out.writeInt(storefileIndexSizeMB);
}
@@ -199,9 +212,11 @@
Integer.valueOf(this.stores));
sb = Strings.appendKeyValue(sb, "storefiles",
Integer.valueOf(this.storefiles));
- sb = Strings.appendKeyValue(sb, "memstoreSize",
+ sb = Strings.appendKeyValue(sb, "storefileSizeMB",
+ Integer.valueOf(this.storefileSizeMB));
+ sb = Strings.appendKeyValue(sb, "memstoreSizeMB",
Integer.valueOf(this.memstoreSizeMB));
- sb = Strings.appendKeyValue(sb, "storefileIndexSize",
+ sb = Strings.appendKeyValue(sb, "storefileIndexSizeMB",
Integer.valueOf(this.storefileIndexSizeMB));
return sb.toString();
}
@@ -333,6 +348,20 @@
}
/**
+ * @returns the amount of heap in use, in MB
+ */
+ public int getUsedHeapMB() {
+ return usedHeapMB;
+ }
+
+ /**
+ * @returns the maximum allowable heap size, in MB
+ */
+ public int getMaxHeapMB() {
+ return maxHeapMB;
+ }
+
+ /**
* @return region load metrics
*/
public Collection<RegionLoad> getRegionsLoad() {
@@ -350,6 +379,16 @@
}
/**
+ * @return Total size of store files in MB
+ */
+ public int getStorefileSizeInMB() {
+ int count = 0;
+ for (RegionLoad info: regionLoad)
+ count += info.getStorefileSizeMB();
+ return count;
+ }
+
+ /**
* @return Size of memstores in MB
*/
public int getMemStoreSizeInMB() {
@@ -417,10 +456,10 @@
*/
@Deprecated
public void addRegionInfo(final byte[] name, final int stores,
- final int storefiles, final int memstoreSizeMB,
- final int storefileIndexSizeMB) {
+ final int storefiles, final int storefileSizeMB,
+ final int memstoreSizeMB, final int storefileIndexSizeMB) {
this.regionLoad.add(new HServerLoad.RegionLoad(name, stores, storefiles,
- memstoreSizeMB, storefileIndexSizeMB));
+ storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB));
}
// Writable
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java?rev=794910&r1=794909&r2=794910&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java Thu Jul 16 23:55:25 2009
@@ -72,7 +72,8 @@
* <li>Version 18: HBASE-1302.</li>
* <li>Version 19: Added getClusterStatus().</li>
* <li>Version 20: Backed Transaction HBase out of HBase core.</li>
+ * <li>Version 21: HBASE-1665.</li>
* </ul>
*/
- public static final long versionID = 20L;
+ public static final long versionID = 21L;
}
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=794910&r1=794909&r2=794910&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Jul 16 23:55:25 2009
@@ -755,18 +755,21 @@
byte[] name = r.getRegionName();
int stores = 0;
int storefiles = 0;
+ int storefileSizeMB = 0;
int memstoreSizeMB = (int)(r.memstoreSize.get()/1024/1024);
int storefileIndexSizeMB = 0;
synchronized (r.stores) {
stores += r.stores.size();
for (Store store: r.stores.values()) {
storefiles += store.getStorefilesCount();
+ storefileSizeMB +=
+ (int)(store.getStorefilesSize()/1024/1024);
storefileIndexSizeMB +=
(int)(store.getStorefilesIndexSize()/1024/1024);
}
}
- return new HServerLoad.RegionLoad(name, stores, storefiles, memstoreSizeMB,
- storefileIndexSizeMB);
+ return new HServerLoad.RegionLoad(name, stores, storefiles,
+ storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB);
}
/**
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=794910&r1=794909&r2=794910&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Thu Jul 16 23:55:25 2009
@@ -222,41 +222,7 @@
* not flushed.
*/
private boolean flushRegion(HRegion region, boolean removeFromQueue) {
- int count = 0;
- boolean triggered = false;
- boolean finished = false;
- while (count++ < (blockingWaitTime / 500)) {
- finished = true;
- for (Store hstore: region.stores.values()) {
- if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
- // only log once
- if (!triggered) {
- LOG.info("Too many store files for region " + region + ": " +
- hstore.getStorefilesCount() + ", requesting compaction and " +
- "waiting");
- this.server.compactSplitThread.compactionRequested(region, getName());
- triggered = true;
- }
- // pending compaction, not finished
- finished = false;
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- }
- if (triggered && finished) {
- LOG.info("Compaction has completed, we waited " + (count * 500) + "ms, "
- + "finishing flush of region " + region);
- break;
- }
- }
- if (triggered && !finished) {
- LOG.warn("Tried to hold up flushing for compactions of region " + region +
- " but have waited longer than " + blockingWaitTime + "ms, continuing");
- }
-
+ checkStoreFileCount(region);
synchronized (regionsInQueue) {
// See comment above for removeFromQueue on why we do not
// take the region out of the set. If removeFromQueue is true, remove it
@@ -297,7 +263,53 @@
return true;
}
-
+
+ /*
+ * If too many store files already, schedule a compaction and pause a while
+ * before going on with compaction.
+ * @param region Region to check.
+ */
+ private void checkStoreFileCount(final HRegion region) {
+ // If catalog region, do not ever hold up writes (isMetaRegion returns
+ // true if ROOT or META region).
+ if (region.getRegionInfo().isMetaRegion()) return;
+
+ int count = 0;
+ boolean triggered = false;
+ boolean finished = false;
+ while (count++ < (blockingWaitTime / 500)) {
+ finished = true;
+ for (Store hstore: region.stores.values()) {
+ if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
+ // only log once
+ if (!triggered) {
+ LOG.info("Too many store files for region " + region + ": " +
+ hstore.getStorefilesCount() + ", requesting compaction and " +
+ "waiting");
+ this.server.compactSplitThread.compactionRequested(region, getName());
+ triggered = true;
+ }
+ // pending compaction, not finished
+ finished = false;
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ }
+ if (triggered && finished) {
+ LOG.info("Compaction has completed, we waited " + (count * 500) + "ms, "
+ + "finishing flush of region " + region);
+ break;
+ }
+ }
+ if (triggered && !finished) {
+ LOG.warn("Tried to hold up flushing for compactions of region " + region +
+ " but have waited longer than " + blockingWaitTime + "ms, continuing");
+ }
+ }
+
/**
* Check if the regionserver's memstore memory usage is greater than the
* limit. If so, flush regions with the biggest memstores until we're down
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=794910&r1=794909&r2=794910&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Jul 16 23:55:25 2009
@@ -1406,6 +1406,22 @@
}
/**
+ * @return The size of the store files, in bytes.
+ */
+ long getStorefilesSize() {
+ long size = 0;
+ for (StoreFile s: storefiles.values()) {
+ Reader r = s.getReader();
+ if (r == null) {
+ LOG.warn("StoreFile " + s + " has a null Reader");
+ continue;
+ }
+ size += r.length();
+ }
+ return size;
+ }
+
+ /**
* @return The size of the store file indexes, in bytes.
*/
long getStorefilesIndexSize() {