You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2012/06/09 05:46:23 UTC
svn commit: r1348318 - in /hbase/branches/0.90: ./
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/util/
src/test/java/org/apache/hadoop/hbase/util/
src/test/java/org/apache/hadoop/hbase/util/hbck/
Author: jxiang
Date: Sat Jun 9 03:46:22 2012
New Revision: 1348318
URL: http://svn.apache.org/viewvc?rev=1348318&view=rev
Log:
HBASE-6173 hbck check specified tables only
Modified:
hbase/branches/0.90/CHANGES.txt
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1348318&r1=1348317&r2=1348318&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Sat Jun 9 03:46:22 2012
@@ -46,6 +46,7 @@ Release 0.90.7 - Unreleased
HBASE-5973 Add ability for potentially long-running IPC calls to abort if client disconnects (backport by David S. Wang)
HBASE-6013 Polish sharp edges from CopyTable
HBASE-5892 [hbck] Refactor parallel WorkItem* to Futures (Andrew Wang)
+ HBASE-6173 hbck check specified tables only
NEW FEATURE
HBASE-5128 [uber hbck] Online automated repair of table integrity and region consistency problems
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1348318&r1=1348317&r2=1348318&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Sat Jun 9 03:46:22 2012
@@ -71,6 +71,9 @@ public class HRegionInfo extends Version
private static final int ENC_SEPARATOR = '.';
public static final int MD5_HEX_LENGTH = 32;
+ // Current TableName
+ private byte[] tableName = null;
+
/**
* Does region name contain its encoded name?
* @param regionName region name
@@ -367,7 +370,18 @@ public class HRegionInfo extends Version
* @return Table name.
*/
public byte[] getTableName() {
- return getTableName(regionName);
+ if (tableName == null || tableName.length == 0) {
+ tableName = getTableName(getRegionName());
+ }
+ return tableName;
+ }
+
+ /**
+ * Get current table name as string
+ * @return string representation of current table
+ */
+ public String getTableNameAsString() {
+ return Bytes.toString(getTableName());
}
/**
Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1348318&r1=1348317&r2=1348318&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Sat Jun 9 03:46:22 2012
@@ -136,7 +136,6 @@ public class HBaseFsck {
public static final long DEFAULT_TIME_LAG = 60000; // default value of 1 minute
public static final long DEFAULT_SLEEP_BEFORE_RERUN = 10000;
private static final int MAX_NUM_THREADS = 50; // #threads to contact regions
- private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
private static boolean rsSupportsOffline = true;
private static final int DEFAULT_OVERLAPS_TO_SIDELINE = 2;
private static final int DEFAULT_MAX_MERGE = 5;
@@ -164,8 +163,9 @@ public class HBaseFsck {
private boolean fixHdfsOrphans = false; // fix fs holes (missing .regioninfo)
private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
- // limit fixes to listed tables, if empty atttempt to fix all
- private List<byte[]> tablesToFix = new ArrayList<byte[]>();
+ // limit checking/fixes to listed tables, if empty attempt to check/fix all
+ // -ROOT- and .META. are always checked
+ private Set<String> tablesIncluded = new HashSet<String>();
private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge
private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline
private boolean sidelineBigOverlaps = false; // sideline overlaps with >maxMerge regions
@@ -197,6 +197,11 @@ public class HBaseFsck {
* This map from tablename -> TableInfo contains the structures necessary to
* detect table consistency problems (holes, dupes, overlaps). It is sorted
* to prevent dupes.
+ *
+ * If tablesIncluded is empty, this map contains all tables.
+ * Otherwise, it contains only meta tables and tables in tablesIncluded,
+ * unless checkMetaOnly is specified, in which case, it contains only
+ * the meta tables (.META. and -ROOT-).
*/
private SortedMap<String, TableInfo> tablesInfo = new ConcurrentSkipListMap<String,TableInfo>();
@@ -654,9 +659,9 @@ public class HBaseFsck {
if (modTInfo == null) {
// only executed once per table.
modTInfo = new TableInfo(tableName);
+ tablesInfo.put(tableName, modTInfo);
}
modTInfo.addRegionInfo(hbi);
- tablesInfo.put(tableName, modTInfo);
}
return tablesInfo;
@@ -809,14 +814,8 @@ public class HBaseFsck {
for (TableInfo tInfo : tablesInfo.values()) {
TableIntegrityErrorHandler handler;
if (fixHoles || fixOverlaps) {
- if (shouldFixTable(Bytes.toBytes(tInfo.getName()))) {
- handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
- fixHoles, fixOverlaps);
- } else {
- LOG.info("Table " + tInfo.getName() + " is not in the include table " +
- "list. Just suggesting fixes.");
- handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
- }
+ handler = tInfo.new HDFSIntegrityFixer(tInfo, errors, conf,
+ fixHoles, fixOverlaps);
} else {
handler = tInfo.new IntegrityFixSuggester(tInfo, errors);
}
@@ -1004,7 +1003,7 @@ public class HBaseFsck {
if (dirName.equals(HConstants.VERSION_FILE_NAME)) {
foundVersionFile = true;
} else {
- if (!checkMetaOnly ||
+ if ((!checkMetaOnly && isTableIncluded(dirName)) ||
dirName.equals("-ROOT-") ||
dirName.equals(".META.")) {
tableDirs.add(file);
@@ -2216,6 +2215,10 @@ public class HBaseFsck {
startCode = value;
}
+ if (!(isTableIncluded(info.getTableNameAsString())
+ || info.isMetaRegion() || info.isRootRegion())) {
+ return true;
+ }
MetaEntry m = new MetaEntry(info, server, startCode, ts);
HbckInfo hbInfo = new HbckInfo(m);
HbckInfo previous = regionInfoMap.put(info.getEncodedName(), hbInfo);
@@ -2244,7 +2247,7 @@ public class HBaseFsck {
// Scan .META. to pick up user regions
MetaScanner.metaScan(conf, visitor);
}
-
+
errors.print("");
return true;
}
@@ -2671,9 +2674,7 @@ public class HBaseFsck {
// list all online regions from this region server
List<HRegionInfo> regions = server.getOnlineRegions();
- if (hbck.checkMetaOnly) {
- regions = filterOnlyMetaRegions(regions);
- }
+ regions = filterRegions(regions);
if (details) {
errors.detail("RegionServer: " + rsinfo.getServerName() +
" number of regions: " + regions.size());
@@ -2699,10 +2700,11 @@ public class HBaseFsck {
return null;
}
- private List<HRegionInfo> filterOnlyMetaRegions(List<HRegionInfo> regions) {
+ private List<HRegionInfo> filterRegions(List<HRegionInfo> regions) {
List<HRegionInfo> ret = Lists.newArrayList();
for (HRegionInfo hri : regions) {
- if (hri.isMetaRegion() || hri.isRootRegion()) {
+ if (hri.isMetaRegion() || hri.isRootRegion() || (!hbck.checkMetaOnly
+ && hbck.isTableIncluded(hri.getTableNameAsString()))) {
ret.add(hri);
}
}
@@ -2963,22 +2965,15 @@ public class HBaseFsck {
}
/**
- * Only fix tables specified by the list
+ * Only check/fix tables specified by the list,
+ * Empty list means all tables are included.
*/
- boolean shouldFixTable(byte[] table) {
- if (tablesToFix.size() == 0) {
- return true;
- }
-
- // doing this naively since byte[] equals may not do what we want.
- for (byte[] t : tablesToFix) {
- if (Bytes.equals(t, table)) return true;
- }
- return false;
+ boolean isTableIncluded(String table) {
+ return (tablesIncluded.size() == 0) || tablesIncluded.contains(table);
}
- void includeTable(byte[] table) {
- tablesToFix.add(table);
+ public void includeTable(String table) {
+ tablesIncluded.add(table);
}
/**
@@ -3140,9 +3135,8 @@ public class HBaseFsck {
} else if (cmd.equals("-metaonly")) {
fsck.setCheckMetaOnly();
} else {
- byte[] table = Bytes.toBytes(cmd);
- fsck.includeTable(table);
- System.out.println("Allow fixes for table: " + cmd);
+ fsck.includeTable(cmd);
+ System.out.println("Allow checking/fixes for table: " + cmd);
}
}
// do the real work of fsck
Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java?rev=1348318&r1=1348317&r2=1348318&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java Sat Jun 9 03:46:22 2012
@@ -966,4 +966,49 @@ public class TestHBaseFsck {
deleteTable(table);
}
}
+
+ /**
+ * This creates two tables and mess both of them and fix them one by one
+ */
+ @Test
+ public void testFixByTable() throws Exception {
+ String table1 = "testFixByTable1";
+ String table2 = "testFixByTable2";
+ try {
+ setupTable(table1);
+ // make sure data in regions, if in hlog only there is no data loss
+ TEST_UTIL.getHBaseAdmin().flush(table1);
+ // Mess them up by leaving a hole in the hdfs data
+ deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+ Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+ setupTable(table2);
+ // make sure data in regions, if in hlog only there is no data loss
+ TEST_UTIL.getHBaseAdmin().flush(table2);
+ // Mess them up by leaving a hole in the hdfs data
+ deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
+ Bytes.toBytes("C"), false, false, true); // don't rm meta
+
+ HBaseFsck hbck = doFsck(conf, false);
+ assertErrors(hbck, new ERROR_CODE[] {
+ ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
+
+ // fix hole in table 1
+ doFsck(conf, true, table1);
+ // check that hole in table 1 fixed
+ assertNoErrors(doFsck(conf, false, table1));
+ // check that hole in table 2 still there
+ assertErrors(doFsck(conf, false, table2),
+ new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
+
+ // fix hole in table 2
+ doFsck(conf, true, table2);
+ // check that hole in both tables fixed
+ assertNoErrors(doFsck(conf, false));
+ assertEquals(ROWKEYS.length - 2, countRows());
+ } finally {
+ deleteTable(table1);
+ deleteTable(table2);
+ }
+ }
}
Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java?rev=1348318&r1=1348317&r2=1348318&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java Sat Jun 9 03:46:22 2012
@@ -28,13 +28,20 @@ import org.apache.hadoop.hbase.util.HBas
import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
public class HbckTestingUtil {
- public static HBaseFsck doFsck(Configuration conf, boolean fix) throws Exception {
- return doFsck(conf, fix, fix, fix, fix,fix, fix);
+ public static HBaseFsck doFsck(
+ Configuration conf, boolean fix) throws Exception {
+ return doFsck(conf, fix, null);
+ }
+
+ public static HBaseFsck doFsck(
+ Configuration conf, boolean fix, String table) throws Exception {
+ return doFsck(conf, fix, fix, fix, fix,fix, fix, table);
}
public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
- boolean fixHdfsOrphans, boolean fixVersionFile) throws Exception {
+ boolean fixHdfsOrphans, boolean fixVersionFile,
+ String table) throws Exception {
HBaseFsck fsck = new HBaseFsck(conf);
fsck.connect();
fsck.setDisplayFullReport(); // i.e. -details
@@ -45,11 +52,13 @@ public class HbckTestingUtil {
fsck.setFixHdfsOverlaps(fixHdfsOverlaps);
fsck.setFixHdfsOrphans(fixHdfsOrphans);
fsck.setFixVersionFile(fixVersionFile);
+ if (table != null) {
+ fsck.includeTable(table);
+ }
fsck.onlineHbck();
return fsck;
}
-
public static void assertNoErrors(HBaseFsck fsck) throws Exception {
List<ERROR_CODE> errs = fsck.getErrors().getErrorList();
assertEquals(new ArrayList<ERROR_CODE>(), errs);