You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jm...@apache.org on 2012/04/12 19:43:27 UTC
svn commit: r1325402 - in /hbase/branches/0.90: ./
src/main/java/org/apache/hadoop/hbase/util/
src/test/java/org/apache/hadoop/hbase/util/
Author: jmhsieh
Date: Thu Apr 12 17:43:26 2012
New Revision: 1325402
URL: http://svn.apache.org/viewvc?rev=1325402&view=rev
Log:
HBASE-5719 Enhance hbck to sideline overlapped mega regions (Jimmy Xiang)
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1325402&r1=1325401&r2=1325402&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Thu Apr 12 17:43:26 2012
@@ -24,6 +24,7 @@ Release 0.90.7 - Unreleased
NEW FEATURE
HBASE-5128 [uber hbck] Online automated repair of table integrity and region consistency problems
HBASE-5599 [hbck] handle NO_VERSION_FILE and SHOULD_NOT_BE_DEPLOYED inconsistencies (fulin wang)
+ HBASE-5719 Enhance hbck to sideline overlapped mega regions (Jimmy Xiang)
Release 0.90.6 - March 16, 2012
INCOMPATIBLE CHANGES
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1325402&r1=1325401&r2=1325402&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Thu Apr 12 17:43:26 2012
@@ -22,8 +22,10 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
@@ -132,6 +134,7 @@ public class HBaseFsck {
private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
private static boolean rsSupportsOffline = true;
+ private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
private static final int DEFAULT_MAX_MERGE = 5;
/**********************
@@ -143,7 +146,6 @@ public class HBaseFsck {
private HBaseAdmin admin;
private HTable meta;
private ThreadPoolExecutor executor; // threads to retrieve data from regionservers
- private int numThreads = MAX_NUM_THREADS;
private long hbckStartMillis = System.currentTimeMillis();
/***********
@@ -161,6 +163,8 @@ public class HBaseFsck {
// limit fixes to listed tables, if empty atttempt to fix all
private List<byte[]> tablesToFix = new ArrayList<byte[]>();
private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
+ private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
+ private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
private boolean rerun = false; // if we tried to fix something, rerun hbck
private static boolean summary = false; // if we want to print less output
@@ -205,8 +209,8 @@ public class HBaseFsck {
public HBaseFsck(Configuration conf) throws MasterNotRunningException,
ZooKeeperConnectionException, IOException {
this.conf = conf;
- numThreads = conf.getInt("hbasefsck.numthreads", numThreads);
- executor = new ThreadPoolExecutor(0, numThreads,
+ int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS);
+ executor = new ThreadPoolExecutor(1, numThreads,
THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>());
}
@@ -804,14 +808,14 @@ public class HBaseFsck {
/**
* Sideline a region dir (instead of deleting it)
*/
- void sidelineRegionDir(FileSystem fs, HbckInfo hi)
+ Path sidelineRegionDir(FileSystem fs, HbckInfo hi)
throws IOException {
String tableName = Bytes.toString(hi.getTableName());
Path regionDir = hi.getHdfsRegionDir();
if (!fs.exists(regionDir)) {
LOG.warn("No previous " + regionDir + " exists. Continuing.");
- return;
+ return null;
}
Path sidelineTableDir= new Path(getSidelineDir(), tableName);
@@ -844,12 +848,15 @@ public class HBaseFsck {
// dst (foo/a) exists and is a dir, and the src (foo/b) is a dir,
// it moves the src into the dst dir resulting in (foo/a/b). If
// the dst does not exist, and the src a dir, src becomes dst. (foo/b)
- for (FileStatus hfile : fs.listStatus(src)) {
- success = fs.rename(hfile.getPath(), dst);
- if (!success) {
- String msg = "Unable to rename file " + src + " to " + dst;
- LOG.error(msg);
- throw new IOException(msg);
+ FileStatus[] hfiles = fs.listStatus(src);
+ if (hfiles != null && hfiles.length > 0) {
+ for (FileStatus hfile : hfiles) {
+ success = fs.rename(hfile.getPath(), dst);
+ if (!success) {
+ String msg = "Unable to rename file " + src + " to " + dst;
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
}
}
LOG.debug("Sideline directory contents:");
@@ -864,6 +871,7 @@ public class HBaseFsck {
LOG.error(msg);
throw new IOException(msg);
}
+ return sidelineRegionDir;
}
/**
@@ -1488,6 +1496,9 @@ public class HBaseFsck {
// backwards regions
final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
+ // sidelined big overlapped regions
+ final Map<Path, HbckInfo> sidelinedRegions = new HashMap<Path, HbckInfo>();
+
// region split calculator
final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
@@ -1531,6 +1542,9 @@ public class HBaseFsck {
private HTableDescriptor getHTD() {
if (htds.size() == 1) {
return (HTableDescriptor)htds.toArray()[0];
+ } else {
+ LOG.error("None/Multiple table descriptors found for table '"
+ + tableName + "' regions: " + htds);
}
return null;
}
@@ -1711,13 +1725,21 @@ public class HBaseFsck {
if (overlap.size() > maxMerge) {
LOG.warn("Overlap group has " + overlap.size() + " overlapping " +
- "regions which is greater than " + maxMerge + ", the max " +
- "number of regions to merge.");
+ "regions which is greater than " + maxMerge + ", the max number of regions to merge");
+ if (sidelineBigOverlaps) {
+ // we only sideline big overlapped groups that exceeds the max number of regions to merge
+ sidelineBigOverlaps(overlap);
+ }
return;
}
+ mergeOverlaps(overlap);
+ }
+
+ void mergeOverlaps(Collection<HbckInfo> overlap)
+ throws IOException {
LOG.info("== Merging regions into one region: "
- + Joiner.on(",").join(overlap));
+ + Joiner.on(",").join(overlap));
// get the min / max range and close all concerned regions
Pair<byte[], byte[]> range = null;
for (HbckInfo hi : overlap) {
@@ -1781,7 +1803,48 @@ public class HBaseFsck {
fixes++;
}
}
- };
+
+ /**
+ * Sideline some regions in a big overlap group so that it
+ * will have fewer regions, and it is easier to merge them later on.
+ *
+ * @param bigOverlap the overlapped group with regions more than maxMerge
+ * @throws IOException
+ */
+ void sidelineBigOverlaps(
+ Collection<HbckInfo> bigOverlap) throws IOException {
+ int overlapsToSideline = bigOverlap.size() - maxMerge;
+ if (overlapsToSideline > maxOverlapsToSideline) {
+ overlapsToSideline = maxOverlapsToSideline;
+ }
+ List<HbckInfo> regionsToSideline =
+ RegionSplitCalculator.findBigRanges(bigOverlap, overlapsToSideline);
+ FileSystem fs = FileSystem.get(conf);
+ for (HbckInfo regionToSideline: regionsToSideline) {
+ try {
+ LOG.info("Closing region: " + regionToSideline);
+ closeRegion(regionToSideline);
+ } catch (InterruptedException ie) {
+ LOG.warn("Was unable to close region " + regionToSideline.getRegionNameAsString()
+ + ". Interrupted.");
+ throw new IOException(ie);
+ }
+
+ LOG.info("Offlining region: " + regionToSideline);
+ offline(regionToSideline.getRegionName());
+
+ LOG.info("Before sideline big overlapped region: " + regionToSideline.toString());
+ Path sidelineRegionDir = sidelineRegionDir(fs, regionToSideline);
+ if (sidelineRegionDir != null) {
+ sidelinedRegions.put(sidelineRegionDir, regionToSideline);
+ LOG.info("After sidelined big overlapped region: "
+ + regionToSideline.getRegionNameAsString()
+ + " to " + sidelineRegionDir.toString());
+ fixes++;
+ }
+ }
+ }
+ }
/**
* Check the region chain of this table. We are looking for holes,
@@ -1866,16 +1929,22 @@ public class HBaseFsck {
if (details) {
// do full region split map dump
- System.out.println("---- Table '" + this.tableName
+ System.out.println("---- Table '" + this.tableName
+ "': region split map");
dump(splits, regions);
- System.out.println("---- Table '" + this.tableName
+ System.out.println("---- Table '" + this.tableName
+ "': overlap groups");
dumpOverlapProblems(overlapGroups);
System.out.println("There are " + overlapGroups.keySet().size()
+ " overlap groups with " + overlapGroups.size()
+ " overlapping regions");
}
+ if (!sidelinedRegions.isEmpty()) {
+ LOG.warn("Sidelined big overlapped regions, please bulk load them!");
+ System.out.println("---- Table '" + this.tableName
+ + "': sidelined big overlapped regions");
+ dumpSidelinedRegions(sidelinedRegions);
+ }
return errors.getErrorList().size() == originalErrorsCount;
}
@@ -1908,6 +1977,13 @@ public class HBaseFsck {
}
}
+ public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) {
+ for (Path k : regions.keySet()) {
+ System.out.println("To be bulk loaded sidelined region dir: "
+ + k.toString());
+ }
+ }
+
public Multimap<byte[], HbckInfo> getOverlapGroups(
String table) {
TableInfo ti = tablesInfo.get(table);
@@ -2736,6 +2812,14 @@ public class HBaseFsck {
return fixVersionFile;
}
+ public void setSidelineBigOverlaps(boolean sbo) {
+ this.sidelineBigOverlaps = sbo;
+ }
+
+ public boolean shouldSidelineBigOverlaps() {
+ return sidelineBigOverlaps;
+ }
+
/**
* @param mm maximum number of regions to merge into a single region.
*/
@@ -2747,6 +2831,14 @@ public class HBaseFsck {
return maxMerge;
}
+ public void setMaxOverlapsToSideline(int mo) {
+ this.maxOverlapsToSideline = mo;
+ }
+
+ public int getMaxOverlapsToSideline() {
+ return maxOverlapsToSideline;
+ }
+
/**
* Only fix tables specified by the list
*/
@@ -2796,8 +2888,10 @@ public class HBaseFsck {
System.err.println(" -fixHdfsOverlaps Try to fix region overlaps in hdfs.");
System.err.println(" -fixVersionFile Try to fix missing hbase.version file in hdfs.");
System.err.println(" -maxMerge <n> When fixing region overlaps, allow at most <n> regions to merge. (n=" + DEFAULT_MAX_MERGE +" by default)");
+ System.err.println(" -sidelineBigOverlaps When fixing region overlaps, allow to sideline big overlaps");
+ System.err.println(" -maxOverlapsToSideline <n> When fixing region overlaps, allow at most <n> regions to sideline per group. (n=" + DEFAULT_OVERLAPS_TO_SIDELINE +" by default)");
System.err.println("");
- System.err.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile");
+ System.err.println(" -repair Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans -fixHdfsOverlaps -fixVersionFile -sidelineBigOverlaps");
System.err.println(" -repairHoles Shortcut for -fixAssignments -fixMeta -fixHdfsHoles -fixHdfsOrphans");
Runtime.getRuntime().exit(-2);
@@ -2862,6 +2956,8 @@ public class HBaseFsck {
fsck.setFixHdfsOverlaps(true);
} else if (cmd.equals("-fixVersionFile")) {
fsck.setFixVersionFile(true);
+ } else if (cmd.equals("-sidelineBigOverlaps")) {
+ fsck.setSidelineBigOverlaps(true);
} else if (cmd.equals("-repair")) {
// this attempts to merge overlapping hdfs regions, needs testing
// under load
@@ -2871,6 +2967,7 @@ public class HBaseFsck {
fsck.setFixAssignments(true);
fsck.setFixHdfsOverlaps(true);
fsck.setFixVersionFile(true);
+ fsck.setSidelineBigOverlaps(true);
} else if (cmd.equals("-repairHoles")) {
// this will make all missing hdfs regions available but may lose data
fsck.setFixHdfsHoles(true);
@@ -2878,6 +2975,20 @@ public class HBaseFsck {
fsck.setFixMeta(true);
fsck.setFixAssignments(true);
fsck.setFixHdfsOverlaps(false);
+ fsck.setSidelineBigOverlaps(false);
+ } else if (cmd.equals("-maxOverlapsToSideline")) {
+ if (i == args.length - 1) {
+ System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+ printUsageAndExit();
+ }
+ try {
+ int maxOverlapsToSideline = Integer.parseInt(args[i+1]);
+ fsck.setMaxOverlapsToSideline(maxOverlapsToSideline);
+ } catch (NumberFormatException e) {
+ System.err.println("-maxOverlapsToSideline needs a numeric value argument.");
+ printUsageAndExit();
+ }
+ i++;
} else if (cmd.equals("-maxMerge")) {
if (i == args.length - 1) {
System.err.println("-maxMerge needs a numeric value argument.");
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java?rev=1325402&r1=1325401&r2=1325402&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/RegionSplitCalculator.java Thu Apr 12 17:43:26 2012
@@ -19,9 +19,12 @@
*/
package org.apache.hadoop.hbase.util;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
+import java.util.List;
import java.util.Map.Entry;
+import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.commons.logging.Log;
@@ -94,7 +97,7 @@ public class RegionSplitCalculator<R ext
*
* @return ENDKEY if end key is empty, else normal endkey.
*/
- private byte[] specialEndKey(R range) {
+ private static <R extends KeyRange> byte[] specialEndKey(R range) {
byte[] end = range.getEndKey();
if (end.length == 0) {
return ENDKEY;
@@ -159,4 +162,75 @@ public class RegionSplitCalculator<R ext
return starts;
}
+ /**
+ * Find specified number of top ranges in a big overlap group.
+ * It could return less if there are not that many top ranges.
+ * Once these top ranges are excluded, the big overlap group will
+ * be broken into ranges with no overlapping, or smaller overlapped
+ * groups, and most likely some holes.
+ *
+ * @param bigOverlap a list of ranges that overlap with each other
+ * @param count the max number of ranges to find
+ * @return a list of ranges that overlap with most others
+ */
+ public static <R extends KeyRange> List<R>
+ findBigRanges(Collection<R> bigOverlap, int count) {
+ List<R> bigRanges = new ArrayList<R>();
+
+ // The key is the count of overlaps,
+ // The value is a list of ranges that have that many overlaps
+ TreeMap<Integer, List<R>> overlapRangeMap = new TreeMap<Integer, List<R>>();
+ for (R r: bigOverlap) {
+ // Calculates the # of overlaps for each region
+ // and populates rangeOverlapMap
+ byte[] startKey = r.getStartKey();
+ byte[] endKey = specialEndKey(r);
+
+ int overlappedRegions = 0;
+ for (R rr: bigOverlap) {
+ byte[] start = rr.getStartKey();
+ byte[] end = specialEndKey(rr);
+
+ if (BYTES_COMPARATOR.compare(startKey, end) < 0
+ && BYTES_COMPARATOR.compare(endKey, start) > 0) {
+ overlappedRegions++;
+ }
+ }
+
+ // One region always overlaps with itself,
+ // so overlappedRegions should be more than 1
+ // for actual overlaps.
+ if (overlappedRegions > 1) {
+ Integer key = Integer.valueOf(overlappedRegions);
+ List<R> ranges = overlapRangeMap.get(key);
+ if (ranges == null) {
+ ranges = new ArrayList<R>();
+ overlapRangeMap.put(key, ranges);
+ }
+ ranges.add(r);
+ }
+ }
+ int toBeAdded = count;
+ for (Integer key: overlapRangeMap.descendingKeySet()) {
+ List<R> chunk = overlapRangeMap.get(key);
+ int chunkSize = chunk.size();
+ if (chunkSize <= toBeAdded) {
+ bigRanges.addAll(chunk);
+ toBeAdded -= chunkSize;
+ if (toBeAdded > 0) continue;
+ } else {
+ // Try to use the middle chunk in case the overlapping is
+ // chained, for example: [a, c), [b, e), [d, g), [f h)...
+ // In such a case, sideline the middle chunk will break
+ // the group efficiently.
+ int start = (chunkSize - toBeAdded)/2;
+ int end = start + toBeAdded;
+ for (int i = start; i < end; i++) {
+ bigRanges.add(chunk.get(i));
+ }
+ }
+ break;
+ }
+ return bigRanges;
+ }
}
Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java?rev=1325402&r1=1325401&r2=1325402&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestRegionSplitCalculator.java Thu Apr 12 17:43:26 2012
@@ -20,9 +20,12 @@
package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
+import java.util.List;
import java.util.SortedSet;
import org.apache.commons.logging.Log;
@@ -347,4 +350,40 @@ public class TestRegionSplitCalculator {
assertEquals(res, ":\t[, A]\t\n" + "A:\t[A, B]\t\n" + "B:\t[B, ]\t\n"
+ "null:\t\n");
}
+
+ @Test
+ public void testBigRanges() {
+ SimpleRange ai = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("I"));
+ SimpleRange ae = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E"));
+ SimpleRange ac = new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C"));
+
+ Collection<SimpleRange> bigOverlap = new ArrayList<SimpleRange>();
+ bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("E")));
+ bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("C")));
+ bigOverlap.add(new SimpleRange(Bytes.toBytes("A"), Bytes.toBytes("B")));
+ bigOverlap.add(new SimpleRange(Bytes.toBytes("B"), Bytes.toBytes("C")));
+ bigOverlap.add(new SimpleRange(Bytes.toBytes("E"), Bytes.toBytes("H")));
+ bigOverlap.add(ai);
+ bigOverlap.add(ae);
+ bigOverlap.add(ac);
+
+ // Expect 1 range to be returned: ai
+ List<SimpleRange> bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 1);
+ assertEquals(1, bigRanges.size());
+ assertEquals(ai, bigRanges.get(0));
+
+ // Expect 3 ranges to be returned: ai, ae and ac
+ bigRanges = RegionSplitCalculator.findBigRanges(bigOverlap, 3);
+ assertEquals(3, bigRanges.size());
+ assertEquals(ai, bigRanges.get(0));
+
+ SimpleRange r1 = bigRanges.get(1);
+ SimpleRange r2 = bigRanges.get(2);
+ assertEquals(Bytes.toString(r1.start), "A");
+ assertEquals(Bytes.toString(r2.start), "A");
+ String r1e = Bytes.toString(r1.end);
+ String r2e = Bytes.toString(r2.end);
+ assertTrue((r1e.equals("C") && r2e.equals("E"))
+ || (r1e.equals("E") && r2e.equals("C")));
+ }
}