You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2013/03/15 21:40:31 UTC
svn commit: r1457102 - in /hbase/branches/0.95/hbase-server/src:
main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
Author: tedyu
Date: Fri Mar 15 20:40:31 2013
New Revision: 1457102
URL: http://svn.apache.org/r1457102
Log:
HBASE-7930 hbck should provide an option to fix .META. rows with empty REGIONINFO_QUALIFIER (Jean-Marc)
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1457102&r1=1457101&r2=1457102&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Fri Mar 15 20:40:31 2013
@@ -22,6 +22,7 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -194,6 +195,7 @@ public class HBaseFsck extends Configure
private boolean fixVersionFile = false; // fix missing hbase.version file in hdfs
private boolean fixSplitParents = false; // fix lingering split parents
private boolean fixReferenceFiles = false; // fix lingering reference store file
+ private boolean fixEmptyMetaCells = false; // fix (remove) empty REGIONINFO_QUALIFIER rows
// limit checking/fixes to listed tables, if empty attempt to check/fix all
// .META. are always checked
@@ -396,6 +398,14 @@ public class HBaseFsck extends Configure
return -2;
}
+ // Empty cells in .META.?
+ reportEmptyMetaCells();
+
+ // Check if we have to cleanup empty REGIONINFO_QUALIFIER rows from .META.
+ if (shouldFixEmptyMetaCells()) {
+ fixEmptyMetaCells();
+ }
+
// get a list of all tables that have not changed recently.
if (!checkMetaOnly) {
reportTablesInFlux();
@@ -410,9 +420,6 @@ public class HBaseFsck extends Configure
loadHdfsRegionInfos();
}
- // Empty cells in .META.?
- reportEmptyMetaCells();
-
// Get disabled tables from ZooKeeper
loadDisabledTables();
@@ -849,6 +856,21 @@ public class HBaseFsck extends Configure
}
/**
+ * To fix the empty REGIONINFO_QUALIFIER rows from .META. <br>
+ * @throws IOException
+ */
+ public void fixEmptyMetaCells() throws IOException {
+ if (shouldFixEmptyMetaCells() && !emptyRegionInfoQualifiers.isEmpty()) {
+ LOG.info("Trying to fix empty REGIONINFO_QUALIFIER .META. rows.");
+ for (Result region : emptyRegionInfoQualifiers) {
+ deleteMetaRegion(region.getRow());
+ errors.getErrorList().remove(ERROR_CODE.EMPTY_META_CELL);
+ }
+ emptyRegionInfoQualifiers.clear();
+ }
+ }
+
+ /**
* To fix orphan table by creating a .tableinfo file under tableDir <br>
* 1. if TableInfo is cached, to recover the .tableinfo accordingly <br>
* 2. else create a default .tableinfo file with following items<br>
@@ -1406,10 +1428,17 @@ public class HBaseFsck extends Configure
* Deletes region from meta table
*/
private void deleteMetaRegion(HbckInfo hi) throws IOException {
- Delete d = new Delete(hi.metaEntry.getRegionName());
+ deleteMetaRegion(hi.metaEntry.getRegionName());
+ }
+
+ /**
+ * Deletes region from meta table
+ */
+ private void deleteMetaRegion(byte[] metaKey) throws IOException {
+ Delete d = new Delete(metaKey);
meta.delete(d);
meta.flushCommits();
- LOG.info("Deleted " + hi.metaEntry.getRegionNameAsString() + " from META" );
+ LOG.info("Deleted " + Bytes.toString(metaKey) + " from META" );
}
/**
@@ -2527,6 +2556,8 @@ public class HBaseFsck extends Configure
Pair<HRegionInfo, ServerName> pair = HRegionInfo.getHRegionInfoAndServerName(result);
if (pair == null || pair.getFirst() == null) {
emptyRegionInfoQualifiers.add(result);
+ errors.reportError(ERROR_CODE.EMPTY_META_CELL,
+ "Empty REGIONINFO_QUALIFIER found in .META.");
return true;
}
ServerName sn = null;
@@ -2600,6 +2631,21 @@ public class HBaseFsck extends Configure
}
return (modTime == me.modTime);
}
+
+ @Override
+ public int hashCode() {
+ int hash = Arrays.hashCode(getRegionName());
+ hash ^= getRegionId();
+ hash ^= Arrays.hashCode(getStartKey());
+ hash ^= Arrays.hashCode(getEndKey());
+ hash ^= Boolean.valueOf(isOffline()).hashCode();
+ hash ^= Arrays.hashCode(getTableName());
+ if (regionServer != null) {
+ hash ^= regionServer.hashCode();
+ }
+ hash ^= modTime;
+ return hash;
+ }
}
/**
@@ -2847,7 +2893,7 @@ public class HBaseFsck extends Configure
FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS,
HOLE_IN_REGION_CHAIN, OVERLAP_IN_REGION_CHAIN, REGION_CYCLE, DEGENERATE_REGION,
ORPHAN_HDFS_REGION, LINGERING_SPLIT_PARENT, NO_TABLEINFO_FILE, LINGERING_REFERENCE_HFILE,
- WRONG_USAGE
+ WRONG_USAGE, EMPTY_META_CELL
}
public void clear();
public void report(String message);
@@ -3221,6 +3267,14 @@ public class HBaseFsck extends Configure
return fixMeta;
}
+ public void setFixEmptyMetaCells(boolean shouldFix) {
+ fixEmptyMetaCells = shouldFix;
+ }
+
+ boolean shouldFixEmptyMetaCells() {
+ return fixEmptyMetaCells;
+ }
+
public void setCheckHdfs(boolean checking) {
checkHdfs = checking;
}
@@ -3407,6 +3461,8 @@ public class HBaseFsck extends Configure
out.println(" -fixSplitParents Try to force offline split parents to be online.");
out.println(" -ignorePreCheckPermission ignore filesystem permission pre-check");
out.println(" -fixReferenceFiles Try to offline lingering reference store files");
+ out.println(" -fixEmptyMetaCells Try to fix .META. entries not referencing any region"
+ + " (empty REGIONINFO_QUALIFIER rows)");
out.println("");
out.println(" Datafile Repair options: (expert features, use with caution!)");
@@ -3529,6 +3585,8 @@ public class HBaseFsck extends Configure
sidelineCorruptHFiles = true;
} else if (cmd.equals("-fixReferenceFiles")) {
setFixReferenceFiles(true);
+ } else if (cmd.equals("-fixEmptyMetaCells")) {
+ setFixEmptyMetaCells(true);
} else if (cmd.equals("-repair")) {
// this attempts to merge overlapping hdfs regions, needs testing
// under load
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java?rev=1457102&r1=1457101&r2=1457102&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java Fri Mar 15 20:40:31 2013
@@ -1309,7 +1309,7 @@ public class TestHBaseFsck {
// TODO: fixHdfsHoles does not work against splits, since the parent dir lingers on
// for some time until children references are deleted. HBCK erroneously sees this as
// overlapping regions
- HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, null);
+ HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, false, null);
assertErrors(hbck, new ERROR_CODE[] {}); //no LINGERING_SPLIT_PARENT reported
// assert that the split META entry is still there.
@@ -1371,7 +1371,7 @@ public class TestHBaseFsck {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN}); //no LINGERING_SPLIT_PARENT
// now fix it. The fix should not revert the region split, but add daughters to META
- hbck = doFsck(conf, true, true, false, false, false, false, false, false, null);
+ hbck = doFsck(conf, true, true, false, false, false, false, false, false, false, null);
assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
@@ -1824,6 +1824,44 @@ public class TestHBaseFsck {
}
/**
+ * Test mission REGIONINFO_QUALIFIER in .META.
+ */
+ @Test
+ public void testMissingRegionInfoQualifier() throws Exception {
+ String table = "testMissingRegionInfoQualifier";
+ try {
+ setupTable(table);
+
+ // Mess it up by removing the RegionInfo for one region.
+ HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName());
+ ResultScanner scanner = meta.getScanner(new Scan());
+ Result result = scanner.next();
+ Delete delete = new Delete (result.getRow());
+ delete.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ meta.delete(delete);
+
+ // Mess it up by creating a fake META entry with no associated RegionInfo
+ meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
+ HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
+ meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
+ HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
+ meta.close();
+
+ HBaseFsck hbck = doFsck(conf, false);
+ assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
+
+ // fix reference file
+ hbck = doFsck(conf, true);
+
+ // check that reference file fixed
+ assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
+ } finally {
+ deleteTable(table);
+ }
+ }
+
+
+ /**
* Test pluggable error reporter. It can be plugged in
* from system property or configuration.
*/
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java?rev=1457102&r1=1457101&r2=1457102&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/HbckTestingUtil.java Fri Mar 15 20:40:31 2013
@@ -38,13 +38,13 @@ public class HbckTestingUtil {
public static HBaseFsck doFsck(
Configuration conf, boolean fix, String table) throws Exception {
- return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, fix, table);
+ return doFsck(conf, fix, fix, fix, fix,fix, fix, fix, fix, fix, table);
}
public static HBaseFsck doFsck(Configuration conf, boolean fixAssignments,
boolean fixMeta, boolean fixHdfsHoles, boolean fixHdfsOverlaps,
boolean fixHdfsOrphans, boolean fixTableOrphans, boolean fixVersionFile,
- boolean fixReferenceFiles, String table) throws Exception {
+ boolean fixReferenceFiles, boolean fixEmptyMetaRegionInfo, String table) throws Exception {
HBaseFsck fsck = new HBaseFsck(conf, exec);
fsck.connect();
fsck.setDisplayFullReport(); // i.e. -details
@@ -57,6 +57,7 @@ public class HbckTestingUtil {
fsck.setFixTableOrphans(fixTableOrphans);
fsck.setFixVersionFile(fixVersionFile);
fsck.setFixReferenceFiles(fixReferenceFiles);
+ fsck.setFixEmptyMetaCells(fixEmptyMetaRegionInfo);
if (table != null) {
fsck.includeTable(table);
}