You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/07/16 00:50:11 UTC
svn commit: r794448 - in /hadoop/hbase/trunk: CHANGES.txt
src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
src/java/org/apache/hadoop/hbase/util/FSUtils.java
Author: stack
Date: Wed Jul 15 22:50:11 2009
New Revision: 794448
URL: http://svn.apache.org/viewvc?rev=794448&view=rev
Log:
HBASE-1662 Tool to run major compaction on catalog regions when hbase is shutdown
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=794448&r1=794447&r2=794448&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Wed Jul 15 22:50:11 2009
@@ -476,10 +476,12 @@
HBASE-1640 Allow passing arguments to jruby script run when run by bin/hbase shell
HBASE-698 HLog recovery is not performed after master failure
HBASE-1643 ScanDeleteTracker takes comparator but it unused
- HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region server
- Some server for region TestTable..." -- deubugging
+ HBASE-1603 MR failed "RetriesExhaustedException: Trying to contact region
+ server Some server for region TestTable..." -- deubugging
HBASE-1470 hbase and HADOOP-4379, dhruba's flush/sync
HBASE-1632 Write documentation for configuring/managing ZooKeeper with HBase
+ HBASE-1662 Tool to run major compaction on catalog regions when hbase is
+ shutdown
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=794448&r1=794447&r2=794448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed Jul 15 22:50:11 2009
@@ -56,6 +56,7 @@
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.ipc.HRegionInterface;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
@@ -1674,14 +1675,14 @@
private byte [] stopRow;
RegionScanner(Scan scan) {
- if(Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
+ if (Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) {
this.stopRow = null;
} else {
this.stopRow = scan.getStopRow();
}
List<KeyValueScanner> scanners = new ArrayList<KeyValueScanner>();
- for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry :
scan.getFamilyMap().entrySet()) {
Store store = stores.get(entry.getKey());
scanners.add(store.getScanner(scan, entry.getValue()));
@@ -1703,21 +1704,20 @@
throws IOException {
// This method should probably be reorganized a bit... has gotten messy
KeyValue kv = this.storeHeap.peek();
- if(kv == null) {
+ if (kv == null) {
return false;
}
byte [] currentRow = kv.getRow();
// See if we passed stopRow
- if(stopRow != null &&
- comparator.compareRows(stopRow, 0, stopRow.length,
- currentRow, 0, currentRow.length)
- <= 0) {
+ if (stopRow != null &&
+ comparator.compareRows(stopRow, 0, stopRow.length,
+ currentRow, 0, currentRow.length) <= 0) {
return false;
}
this.storeHeap.next(results);
while(true) {
kv = this.storeHeap.peek();
- if(kv == null) {
+ if (kv == null) {
return false;
}
byte [] row = kv.getRow();
@@ -2348,4 +2348,106 @@
}
return heapSize;
}
-}
+
+ /*
+ * This method calls System.exit.
+ * @param message Message to print out. May be null.
+ */
+ private static void printUsageAndExit(final String message) {
+ if (message != null && message.length() > 0) System.out.println(message);
+ System.out.println("Usage: HRegion CATLALOG_TABLE_DIR [major_compact]");
+ System.out.println("Options:");
+ System.out.println(" major_compact Pass this option to major compact " +
+ "passed region.");
+ System.out.println("Default outputs scan of passed region.");
+ System.exit(1);
+ }
+
+ /*
+ * Process table.
+ * Do major compaction or list content.
+ * @param fs
+ * @param p
+ * @param log
+ * @param c
+ * @param majorCompact
+ * @throws IOException
+ */
+ private static void processTable(final FileSystem fs, final Path p,
+ final HLog log, final HBaseConfiguration c,
+ final boolean majorCompact)
+ throws IOException {
+ HRegion region = null;
+ String rootStr = Bytes.toString(HConstants.ROOT_TABLE_NAME);
+ String metaStr = Bytes.toString(HConstants.META_TABLE_NAME);
+ // Currently expects tables have one region only.
+ if (p.getName().startsWith(rootStr)) {
+ region = new HRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO, null);
+ } else if (p.getName().startsWith(metaStr)) {
+ region = new HRegion(p, log, fs, c, HRegionInfo.FIRST_META_REGIONINFO,
+ null);
+ } else {
+ throw new IOException("Not a known catalog table: " + p.toString());
+ }
+ try {
+ region.initialize(null, null);
+ if (majorCompact) {
+ region.compactStores(true);
+ } else {
+ // Default behavior
+ Scan scan = new Scan();
+ InternalScanner scanner = region.getScanner(scan);
+ try {
+ List<KeyValue> kvs = new ArrayList<KeyValue>();
+ boolean done = false;
+ do {
+ kvs.clear();
+ done = scanner.next(kvs);
+ if (kvs.size() > 0) LOG.info(kvs);
+ } while (done);
+ } finally {
+ scanner.close();
+ }
+ }
+ } finally {
+ region.close();
+ }
+ }
+
+ /**
+ * Facility for dumping and compacting catalog tables.
+ * Only does catalog tables since these are only tables we for sure know
+ * schema on. For usage run:
+ * <pre>
+ * ./bin/hbase org.apache.hadoop.hbase.regionserver.HRegion
+ * </pre>
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ if (args.length < 1) {
+ printUsageAndExit(null);
+ }
+ boolean majorCompact = false;
+ if (args.length > 1) {
+ if (!args[1].toLowerCase().startsWith("major")) {
+ printUsageAndExit("ERROR: Unrecognized option <" + args[1] + ">");
+ }
+ majorCompact = true;
+
+ }
+ Path tableDir = new Path(args[0]);
+ HBaseConfiguration c = new HBaseConfiguration();
+ FileSystem fs = FileSystem.get(c);
+ Path logdir = new Path(c.get("hbase.tmp.dir"),
+ "hlog" + tableDir.getName() + System.currentTimeMillis());
+ HLog log = new HLog(fs, logdir, c, null);
+ try {
+ processTable(fs, tableDir, log, c, majorCompact);
+ } finally {
+ log.close();
+ BlockCache bc = StoreFile.getBlockCache(c);
+ if (bc != null) bc.shutdown();
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java?rev=794448&r1=794447&r2=794448&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/FSUtils.java Wed Jul 15 22:50:11 2009
@@ -274,29 +274,61 @@
* one file in them -- that is, they've been major compacted. Looks
* at root and meta tables too.
* @param fs
- * @param c
+ * @param hbaseRootDir
* @return True if this hbase install is major compacted.
* @throws IOException
*/
public static boolean isMajorCompacted(final FileSystem fs,
- final HBaseConfiguration c)
+ final Path hbaseRootDir)
throws IOException {
// Presumes any directory under hbase.rootdir is a table.
- FileStatus [] directories =
- fs.listStatus(new Path(c.get(HConstants.HBASE_DIR)), new PathFilter() {
- public boolean accept(Path p) {
- boolean isdir = false;
- try {
- isdir = fs.getFileStatus(p).isDir();
- } catch (IOException e) {
- e.printStackTrace();
+ FileStatus [] directories = fs.listStatus(hbaseRootDir, new DirFilter(fs));
+ for (int i = 0; i < directories.length; i++) {
+ // Skip the .log directory. All others should be tables. Inside a table,
+ // there are compaction.dir directories to skip. Otherwise, all else
+ // should be regions. Then in each region, should only be family
+ // directories. Under each of these, should be one file only.
+ Path d = directories[i].getPath();
+ if (d.getName().equals(HConstants.HREGION_LOGDIR_NAME)) continue;
+ FileStatus [] tablesubdirectories = fs.listStatus(d, new DirFilter(fs));
+ for (int j = 0; j < tablesubdirectories.length; j++) {
+ Path dd = tablesubdirectories[j].getPath();
+ if (dd.equals(HConstants.HREGION_COMPACTIONDIR_NAME)) continue;
+ // Else its a region name. Now look in region for families.
+ FileStatus [] familydirectories = fs.listStatus(dd, new DirFilter(fs));
+ for (int k = 0; k < familydirectories.length; k++) {
+ Path family = familydirectories[k].getPath();
+ // Now in family make sure only one file.
+ FileStatus [] familyStatus = fs.listStatus(family);
+ if (familyStatus.length > 1) {
+ LOG.debug(family.toString() + " has " + familyStatus.length +
+ " files.");
+ return false;
}
- return isdir;
}
- });
- for (int i = 0; i < directories.length; i++) {
- // TODO: check directories
+ }
}
return true;
}
+
+ /**
+ * A {@link PathFilter} that returns directories.
+ */
+ public static class DirFilter implements PathFilter {
+ private final FileSystem fs;
+
+ public DirFilter(final FileSystem fs) {
+ this.fs = fs;
+ }
+
+ public boolean accept(Path p) {
+ boolean isdir = false;
+ try {
+ isdir = this.fs.getFileStatus(p).isDir();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ return isdir;
+ }
+ }
}
\ No newline at end of file