You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2012/05/30 22:23:49 UTC
svn commit: r1344447 - in /hbase/branches/0.90: ./
src/main/java/org/apache/hadoop/hbase/client/
src/main/java/org/apache/hadoop/hbase/ipc/
src/main/java/org/apache/hadoop/hbase/regionserver/
src/main/resources/hbase-webapps/master/
Author: stack
Date: Wed May 30 20:23:49 2012
New Revision: 1344447
URL: http://svn.apache.org/viewvc?rev=1344447&view=rev
Log:
HBASE-6124 Backport HBASE-6033 to 0.90, 0.92 and 0.94
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java
hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Wed May 30 20:23:49 2012
@@ -32,6 +32,7 @@ Release 0.90.7 - Unreleased
HBASE-5922 HalfStoreFileReader seekBefore causes StackOverflowError (Nate Putnam)
HBASE-5757 TableInputFormat should handle as many errors as possible (Jan Lukavsky)
HBASE-6041 NullPointerException prevents the master from starting up (Jimmy Xiang)
+ HBASE-6124 https://issues.apache.org/jira/secure/attachment/12530072/patch-0.90.txt (Jimmy Xiang)
IMPROVEMENT
HBASE-5588 Deprecate/remove AssignmentManager#clearRegionFromTransition
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Wed May 30 20:23:49 2012
@@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.catalog.M
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
import org.apache.hadoop.hbase.ipc.HMasterInterface;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
+import org.apache.hadoop.hbase.regionserver.CompactSplitThread.CompactionState;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Writables;
@@ -1349,4 +1350,88 @@ public class HBaseAdmin implements Abort
this.connection.close();
}
}
+ /**
+ * Get the current compaction state of a table or region.
+ * It could be in a major compaction, a minor compaction, both, or none.
+ *
+ * @param tableNameOrRegionName table or region to major compact
+ * @throws IOException if a remote or network exception occurs
+ * @throws InterruptedException
+ * @return the current compaction state
+ */
+ public CompactionState getCompactionState(final String tableNameOrRegionName)
+ throws IOException, InterruptedException {
+ return getCompactionState(Bytes.toBytes(tableNameOrRegionName));
+ }
+
+ /**
+ * Get the current compaction state of a table or region.
+ * It could be in a major compaction, a minor compaction, both, or none.
+ *
+ * @param tableNameOrRegionName table or region to major compact
+ * @throws IOException if a remote or network exception occurs
+ * @throws InterruptedException
+ * @return the current compaction state
+ */
+ public CompactionState getCompactionState(final byte [] tableNameOrRegionName)
+ throws IOException, InterruptedException {
+ CompactionState state = CompactionState.NONE;
+ CatalogTracker ct = getCatalogTracker();
+ try {
+ if (isRegionName(tableNameOrRegionName, ct)) {
+ Pair<HRegionInfo, HServerAddress> pair =
+ MetaReader.getRegion(ct, tableNameOrRegionName);
+ if (pair == null || pair.getSecond() == null) {
+ LOG.info("No server in .META. for " +
+ Bytes.toStringBinary(tableNameOrRegionName) + "; pair=" + pair);
+ } else {
+ HRegionInterface rs =
+ this.connection.getHRegionConnection(pair.getSecond());
+ return CompactionState.valueOf(
+ rs.getCompactionState(pair.getFirst().getRegionName()));
+ }
+ } else {
+ final String tableName = tableNameString(tableNameOrRegionName, ct);
+ List<Pair<HRegionInfo, HServerAddress>> pairs =
+ MetaReader.getTableRegionsAndLocations(ct, tableName);
+ for (Pair<HRegionInfo, HServerAddress> pair: pairs) {
+ if (pair.getFirst().isOffline()) continue;
+ if (pair.getSecond() == null) continue;
+ try {
+ HRegionInterface rs =
+ this.connection.getHRegionConnection(pair.getSecond());
+ switch (CompactionState.valueOf(
+ rs.getCompactionState(pair.getFirst().getRegionName()))) {
+ case MAJOR_AND_MINOR:
+ return CompactionState.MAJOR_AND_MINOR;
+ case MAJOR:
+ if (state == CompactionState.MINOR) {
+ return CompactionState.MAJOR_AND_MINOR;
+ }
+ state = CompactionState.MAJOR;
+ break;
+ case MINOR:
+ if (state == CompactionState.MAJOR) {
+ return CompactionState.MAJOR_AND_MINOR;
+ }
+ state = CompactionState.MINOR;
+ break;
+ case NONE:
+ default:
+ // nothing, continue
+ }
+ } catch (NotServingRegionException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to get compaction state of " +
+ pair.getFirst() + ": " +
+ StringUtils.stringifyException(e));
+ }
+ }
+ }
+ }
+ } finally {
+ cleanupCatalogTracker(ct);
+ }
+ return state;
+ }
}
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java Wed May 30 20:23:49 2012
@@ -433,4 +433,13 @@ public interface HRegionInterface extend
/* TODO: Move into place above master operations after deprecation cycle */
public boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, byte[] regionName)
throws IOException;
+
+ /**
+ * Get the current compaction state of the region.
+ *
+ * @param regionName the name of the region to check compaction statte.
+ * @return the compaction state name.
+ * @throws IOException exception
+ */
+ public String getCompactionState(final byte[] regionName) throws IOException;
}
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Wed May 30 20:23:49 2012
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
@@ -49,6 +50,53 @@ public class CompactSplitThread extends
public static final int PRIORITY_USER = 1;
/**
+ * Map to track the state of compactions requested per region (id)
+ */
+ private static final ConcurrentHashMap<Long, CompactionState>
+ compactionStates = new ConcurrentHashMap<Long, CompactionState>();
+
+ /**
+ * Find out if a given region is in compaction now.
+ * This information is not accurate in case one request is
+ * being processed while new request comes. So it is just
+ * an indication for normal scenario.
+ *
+ * @param regionId
+ * @return
+ */
+ public static CompactionState getCompactionState(
+ final long regionId) {
+ Long key = Long.valueOf(regionId);
+ CompactionState state = compactionStates.get(key);
+ if (state == null) {
+ state = CompactionState.NONE;
+ }
+ return state;
+ }
+
+ public static void preRequest(final HRegion region){
+ Long key = Long.valueOf(region.getRegionId());
+ CompactionState state =
+ region.getForceMajorCompaction() ? CompactionState.MAJOR : CompactionState.MINOR;
+ compactionStates.put(key, state);
+ }
+
+ public static void postRequest(final HRegion region){
+ Long key = Long.valueOf(region.getRegionId());
+ compactionStates.remove(key);
+ }
+
+ /**
+ * An enum for the region compaction state
+ */
+ public static enum CompactionState {
+ NONE,
+ MINOR,
+ MAJOR,
+ MAJOR_AND_MINOR;
+ }
+
+ /**
* Splitting should not take place if the total number of regions exceed this.
* This is not a hard limit to the number of regions but it is a guideline to
* stop splitting after number of online regions is greater than this.
@@ -89,6 +137,7 @@ public class CompactSplitThread extends
}
} finally {
lock.unlock();
+ CompactSplitThread.postRequest(r);
}
}
} catch (InterruptedException ex) {
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed May 30 20:23:49 2012
@@ -2045,6 +2045,32 @@ public class HRegion implements HeapSize
return this.stores.get(column);
}
+ /**
+ * Return list of storeFiles for the set of CFs.
+ * Uses closeLock to prevent the race condition where a region closes
+ * in between the for loop - closing the stores one by one, some stores
+ * will return 0 files.
+ * @return List of storeFiles.
+ */
+ public List<String> getStoreFileList(final byte [][] columns)
+ throws IllegalArgumentException {
+ List<String> storeFileNames = new ArrayList<String>();
+ synchronized(closeLock) {
+ for(byte[] column : columns) {
+ Store store = this.stores.get(column);
+ if (store == null) {
+ throw new IllegalArgumentException("No column family : " +
+ new String(column) + " available");
+ }
+ List<StoreFile> storeFiles = store.getStorefiles();
+ for (StoreFile storeFile: storeFiles) {
+ storeFileNames.add(storeFile.getPath().toString());
+ }
+ }
+ }
+ return storeFileNames;
+ }
+
//////////////////////////////////////////////////////////////////////////////
// Support code
//////////////////////////////////////////////////////////////////////////////
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Wed May 30 20:23:49 2012
@@ -2798,4 +2798,42 @@ public class HRegionServer implements HR
new HRegionServerCommandLine(regionServerClass).doMain(args);
}
+
+ /**
+ * Gets the online regions of the specified table.
+ * This method looks at the in-memory onlineRegions. It does not go to <code>.META.</code>.
+ * Only returns <em>online</em> regions. If a region on this table has been
+ * closed during a disable, etc., it will not be included in the returned list.
+ * So, the returned list may not necessarily be ALL regions in this table, its
+ * all the ONLINE regions in the table.
+ * @param tableName
+ * @return Online regions from <code>tableName</code>
+ */
+ public List<HRegion> getOnlineRegions(byte[] tableName) {
+ List<HRegion> tableRegions = new ArrayList<HRegion>();
+ synchronized (this.onlineRegions) {
+ for (HRegion region: this.onlineRegions.values()) {
+ HRegionInfo regionInfo = region.getRegionInfo();
+ if(Bytes.equals(regionInfo.getTableName(), tableName)) {
+ tableRegions.add(region);
+ }
+ }
+ }
+ return tableRegions;
+ }
+
+ /**
+ * Get the current compaction state of the region.
+ *
+ * @param regionName the name of the region to check compaction statte.
+ * @return the compaction state name.
+ * @throws IOException exception
+ */
+ public String getCompactionState(final byte[] regionName) throws IOException {
+ checkOpen();
+ requestCount.incrementAndGet();
+ HRegion region = getRegion(regionName);
+ HRegionInfo info = region.getRegionInfo();
+ return CompactSplitThread.getCompactionState(info.getRegionId()).name();
+ }
}
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/regionserver/PriorityCompactionQueue.java Wed May 30 20:23:49 2012
@@ -20,7 +20,6 @@
package org.apache.hadoop.hbase.regionserver;
import java.util.Collection;
-import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.concurrent.BlockingQueue;
@@ -44,7 +43,7 @@ public class PriorityCompactionQueue imp
* This class represents a compaction request and holds the region, priority,
* and time submitted.
*/
- private class CompactionRequest implements Comparable<CompactionRequest> {
+ static class CompactionRequest implements Comparable<CompactionRequest> {
private final HRegion r;
private final int p;
private final Long timeInNanos;
@@ -144,6 +143,7 @@ public class PriorityCompactionQueue imp
newRequest.getPriority() < queuedRequest.getPriority()) {
LOG.trace("Inserting region in queue. " + newRequest);
regionsInQueue.put(r, newRequest);
+ CompactSplitThread.preRequest(r);
} else {
LOG.trace("Region already in queue, skipping. Queued: " + queuedRequest +
", requested: " + newRequest);
@@ -187,8 +187,7 @@ public class PriorityCompactionQueue imp
public boolean add(HRegion e, int p) {
CompactionRequest request = this.addToRegionsInQueue(e, p);
if (request != null) {
- boolean result = queue.add(request);
- return result;
+ return queue.add(request);
} else {
return false;
}
Modified: hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp?rev=1344447&r1=1344446&r2=1344447&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp (original)
+++ hbase/branches/0.90/src/main/resources/hbase-webapps/master/table.jsp Wed May 30 20:23:49 2012
@@ -146,6 +146,11 @@
<td><%= hbadmin.isTableEnabled(table.getTableName()) %></td>
<td>Is the table enabled</td>
</tr>
+ <tr>
+ <td>Compaction</td>
+ <td><%= hbadmin.getCompactionState(table.getTableName()) %></td>
+ <td>Is the table compacting</td>
+ </tr>
<% if (showFragmentation) { %>
<tr>
<td>Fragmentation</td>