You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2014/04/22 20:18:45 UTC
svn commit: r1589229 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
Author: liyin
Date: Tue Apr 22 18:18:45 2014
New Revision: 1589229
URL: http://svn.apache.org/r1589229
Log:
[HBASE-11021] HBaseFsck Fix for Table Hole Reporting
Author: achayes
Summary:
HBaseFsck would not report when multiple regions went missing. Instead, it
would report the first missing region it found in the meta table and exit.
The new implementation now lists all ranges of [startkey, endkey] that are
missing from the meta table, which should allow for easier debugging.
Test Plan:
TestHBaseFsck is a short test file written to test this functionality. It
sets up a minicluster, creates a table, and then deletes a bunch of rows
from the table and runs the checker. Since the checker does not throw errors
on failure, the output logs have to be checked manually to see if there was
any error. The result looks like this:
https://www.facebook.com/pxlcld/l7xF
Reviewers: manukranthk
Reviewed by: manukranthk
CC:
Differential Revision: https://phabricator.fb.com/D1266615
Task ID: #3675218
Added:
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java?rev=1589229&r1=1589228&r2=1589229&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HBaseFsck.java Tue Apr 22 18:18:45 2014
@@ -147,6 +147,14 @@ public class HBaseFsck {
return this.regionInfo;
}
+ public int getTotalHolesFound() {
+ int numHoles = 0;
+ for (TInfo tInfo : this.tablesInfo.values()) {
+ numHoles += tInfo.getHolesFound();
+ }
+ return numHoles;
+ }
+
public int initAndScanRootMeta() throws IOException {
// print hbase server version
errors.print("Version: " + status.getHBaseVersion());
@@ -604,6 +612,7 @@ public class HBaseFsck {
modTInfo.addServer(server);
}
modTInfo.addEdge(hbi.metaEntry.getStartKey(), hbi.metaEntry.getEndKey());
+ modTInfo.addRegionStart(hbi.metaEntry.getStartKey());
tablesInfo.put(tableName, modTInfo);
}
@@ -632,8 +641,10 @@ public class HBaseFsck {
private class TInfo {
String tableName;
TreeMap <byte[], byte[]> edges;
+ ArrayList<byte[]> regionStartKeys;
TreeSet <HServerAddress> deployedOn;
String lastError = null;
+ int holesFound = 0;
private TreeMap<RegionType, ArrayList<String>> regionDetails = new TreeMap<RegionType, ArrayList<String>>();
private ArrayList<String> regionErrors = new ArrayList<String>();
@@ -642,6 +653,7 @@ public class HBaseFsck {
this.tableName = name;
edges = new TreeMap <byte[], byte[]> (Bytes.BYTES_COMPARATOR);
deployedOn = new TreeSet <HServerAddress>();
+ regionStartKeys = new ArrayList<byte[]>();
for (RegionType regionType : RegionType.values()) {
regionDetails.put(regionType, new ArrayList<String>());
}
@@ -651,6 +663,10 @@ public class HBaseFsck {
this.edges.put(fromNode, toNode);
}
+ public void addRegionStart(byte[] key) {
+ this.regionStartKeys.add(key);
+ }
+
public void addServer(HServerAddress server) {
this.deployedOn.add(server);
}
@@ -663,6 +679,10 @@ public class HBaseFsck {
return edges.size();
}
+ public int getHolesFound() {
+ return this.holesFound;
+ }
+
public String getLastError() {
return this.lastError;
}
@@ -682,48 +702,55 @@ public class HBaseFsck {
errors.detail('\t' + regionToStr(e));
}
}
+ StringBuilder errorSB = new StringBuilder("\n");
+ if (this.regionStartKeys.size() == 0) {
+ errors.detail("No regions found in META for " + this.tableName);
+ return true;
+ }
- byte[] last = new byte[0];
- byte[] next = new byte[0];
+ Collections.sort(this.regionStartKeys, Bytes.BYTES_COMPARATOR);
TreeSet <byte[]> visited = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
- // Each table should start with a zero-length byte[] and end at a
- // zero-length byte[]. Just follow the edges to see if this is true
- while (true) {
- // Check if region chain is broken
- if (!edges.containsKey(last)) {
- this.lastError = "Cannot find region with start key "
- + posToStr(last);
- return false;
- }
- next = edges.get(last);
- // Found a cycle
- if (visited.contains(next)) {
- this.lastError = "Cycle found in region chain. "
- + "Current = "+ posToStr(last)
- + "; Cycle Start = " + posToStr(next);
- return false;
- }
- // Mark next node as visited
- visited.add(next);
- // If next is zero-length byte[] we are possibly at the end of the chain
- if (next.length == 0) {
- // If we have visited all elements we are fine
- if (edges.size() != visited.size()) {
- this.lastError = "Region in-order travesal does not include "
- + "all elements found in META. Chain=" + visited.size()
- + "; META=" + edges.size() + "; Missing=";
- for (Map.Entry<byte[], byte []> e : edges.entrySet()) {
- if (!visited.contains(e.getKey())) {
- this.lastError += regionToStr(e) + " , ";
- }
- }
- return false;
- }
- return true;
+
+ // First region should start and end with a zero length byte[].
+ // We will go through each start key in order to see if the chain of
+ // regions is broken at any point and identify any holes that exist
+ byte[] prevEndKey = new byte[0];
+ for (int i = 0; i < this.regionStartKeys.size(); i++) {
+ byte[] curStartKey = this.regionStartKeys.get(i);
+ if (Bytes.compareTo(prevEndKey, curStartKey) != 0) {
+ this.holesFound++;
+ errorSB.append("Found hole in table from start key "
+ + posToStr(prevEndKey) + " to end key " + posToStr(curStartKey) + "\n");
+ }
+
+ visited.add(curStartKey);
+ prevEndKey = this.edges.get(curStartKey);
+ // Checks for cycles in the region chain, including a self cycle
+ // (where the start key equals the end key)
+ if (visited.contains(prevEndKey)
+ && Bytes.compareTo(prevEndKey, new byte[0]) != 0) {
+ errorSB.append("Cycle found in region chain. "
+ + "Current Start Key "+ posToStr(curStartKey)
+ + "; Cycle Start Key " + posToStr(prevEndKey) + "\n");
}
- last = next;
}
- // How did we get here?
+
+ // Check if there's a hole at the end of the table
+ if (Bytes.compareTo(prevEndKey, new byte[0]) != 0) {
+ this.holesFound++;
+ errorSB.append("Found hole in table from start key "
+ + posToStr(prevEndKey) + " to end of table (end key 0)\n");
+ }
+
+ if (this.holesFound > 0) {
+ errorSB.append("Total number of holes found:" + holesFound + "\n");
+ }
+
+ boolean noErrors = errorSB.length() == 0;
+ if (!noErrors) {
+ this.lastError = errorSB.toString();
+ }
+ return noErrors;
}
public JSONObject toJSONObject() {
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java?rev=1589229&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHBaseFsck.java Tue Apr 22 18:18:45 2014
@@ -0,0 +1,158 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map.Entry;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableExistsException;
+import org.apache.hadoop.hbase.TableNotDisabledException;
+import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+/**
+ * Class to test HBaseFsck.
+ */
+public class TestHBaseFsck {
+ final Log LOG = LogFactory.getLog(getClass());
+ private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private HBaseAdmin admin;
+ private static final int NUM_REGION_SERVER = 3;
+
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+ TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
+ TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
+ TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
+ TEST_UTIL.startMiniCluster(NUM_REGION_SERVER);
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ this.admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
+ }
+
+
+ @Test
+ public void testFindMultipleHoles() throws IOException, InterruptedException {
+
+ byte[] tableName = Bytes.toBytes("testCreateTableWithRegions");
+
+ byte [][] splitKeys = {
+ new byte [] { 1, 1, 1 },
+ new byte [] { 2, 2, 2 },
+ new byte [] { 3, 3, 3 },
+ new byte [] { 4, 4, 4 },
+ new byte [] { 5, 5, 5 },
+ new byte [] { 6, 6, 6 },
+ new byte [] { 7, 7, 7 },
+ new byte [] { 8, 8, 8 },
+ new byte [] { 9, 9, 9 },
+ };
+ int expectedRegions = splitKeys.length + 1;
+
+ HTableDescriptor desc = new HTableDescriptor(tableName);
+ desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
+ admin.createTable(desc, splitKeys);
+
+ Configuration conf = TEST_UTIL.getConfiguration();
+ HTable ht = new HTable(conf, tableName);
+ HTableDescriptor htd = ht.getTableDescriptor();
+ Map<HRegionInfo,HServerAddress> regions = ht.getRegionsInfo();
+ System.out.println("Deleting a few rows from HDFS and META");
+
+ deleteRegion(conf, regions, htd, new byte[] {2,2,2}, new byte[] {3,3,3});
+ deleteRegion(conf, regions, htd, new byte[] {3,3,3}, new byte[] {4,4,4});
+ deleteRegion(conf, regions, htd, new byte[] {5,5,5}, new byte[] {6,6,6});
+ deleteRegion(conf, regions, htd, new byte[] {8,8,8}, new byte[] {9,9,9});
+ deleteRegion(conf, regions, htd, new byte[] {9,9,9}, new byte[0] );
+
+ System.err.println("Found " + regions.size() + " regions");
+
+ HBaseFsck fsck = new HBaseFsck(TEST_UTIL.getConfiguration());
+ fsck.displayFullReport();
+ fsck.setTimeLag(0);
+ fsck.doWork();
+ assertEquals(fsck.getTotalHolesFound(), 3);
+ }
+
+ private void deleteRegion(Configuration conf, Map<HRegionInfo,HServerAddress> regions,
+ HTableDescriptor htd, byte[] startKey, byte[] endKey)
+ throws IOException, InterruptedException {
+
+ for (Entry<HRegionInfo, HServerAddress> e: regions.entrySet()) {
+ HRegionInfo hri = e.getKey();
+ HServerAddress hsa = e.getValue();
+ if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
+ && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
+
+ byte[] deleteRow = hri.getRegionName();
+ System.out.println("deleting hdfs data: " + hri.toString() + hsa.toString());
+ Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+ FileSystem fs = rootDir.getFileSystem(conf);
+ Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
+ fs.delete(p, true);
+
+ HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
+ Delete delete = new Delete(deleteRow);
+ meta.delete(delete);
+ }
+ System.out.println(hri.toString() + hsa.toString());
+ }
+
+ TEST_UTIL.getMetaTableRows(htd.getName());
+ }
+
+}