You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2010/10/09 21:23:36 UTC
svn commit: r1006219 - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/catalog/
main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/util/
Author: stack
Date: Sat Oct 9 19:23:36 2010
New Revision: 1006219
URL: http://svn.apache.org/viewvc?rev=1006219&view=rev
Log:
HBASE-3100 TestMergeTable failing in TRUNK
M src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Some fixup to support stepped start up of mini cluster; allow
starting dfs cluster, then later put an hbase mini cluster on top.
(startMiniHBaseCluster, createRootDir): Added.
D src/test/java/org/apache/hadoop/hbase/AbstractMergeTestBase.java
Removed messy subclass of HBaseClusterTestCase used building
up some specific loaded regions. Replaced with utility added
to HBaseTestingUtility and by methods added to specific test.
D src/test/java/org/apache/hadoop/hbase/util/TestMergeMeta.java
Deleted test that did nothing -- test merging of .META. -- but
the superclass was making user regions, not multiple instances
of .META. -- which we don't support anyways currently.
M src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
Rewritten to use HBaseTestingUtility. Also added assertions that
it actually did successful merge (Were none previous).
M src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
Added a new constructor. Are the others redundant given I just
added implementation of Abortable to HConnection interface (the
implmementation of HConnection used implement it -- I've just
moved it up into the Interface itself).
M src/main/java/org/apache/hadoop/hbase/util/HMerge.java
Bit of minor cleanup refactoring.
M src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
The HConnection Interface now implements Abortable.
M src/main/java/org/apache/hadoop/hbase/client/HConnection.java
Extend Abortable (The implementation was implementing Abortable
anyways).
Removed:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/AbstractMergeTestBase.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeMeta.java
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java?rev=1006219&r1=1006218&r2=1006219&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java Sat Oct 9 19:23:36 2010
@@ -71,6 +71,18 @@ public class CatalogTracker {
HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
/**
+ * Constructs a catalog tracker. Find current state of catalog tables and
+ * begin active tracking by executing {@link #start()} post construction.
+ * Does not timeout.
+ * @param connection Server connection; if problem, this connections
+ * {@link HConnection#abort(String, Throwable)} will be called.
+ * @throws IOException
+ */
+ public CatalogTracker(final HConnection connection) throws IOException {
+ this(connection.getZooKeeperWatcher(), connection, connection);
+ }
+
+ /**
* Constructs the catalog tracker. Find current state of catalog tables and
* begin active tracking by executing {@link #start()} post construction.
* Does not timeout.
@@ -274,7 +286,7 @@ public class CatalogTracker {
* for up to the specified timeout if not immediately available. Throws an
* exception if timed out waiting. This method differs from {@link #waitForMeta()}
* in that it will go ahead and verify the location gotten from ZooKeeper by
- * trying trying to use returned connection.
+ * trying to use returned connection.
* @param timeout maximum time to wait for meta availability, in milliseconds
* @return location of meta
* @throws InterruptedException if interrupted while waiting
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=1006219&r1=1006218&r2=1006219&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Sat Oct 9 19:23:36 2010
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
+import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HServerAddress;
@@ -39,7 +40,7 @@ import org.apache.hadoop.hbase.zookeeper
* Cluster connection.
* {@link HConnectionManager} manages instances of this class.
*/
-public interface HConnection {
+public interface HConnection extends Abortable {
/**
* Retrieve ZooKeeperWatcher used by the connection.
* @return ZooKeeperWatcher handle being used by the connection.
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1006219&r1=1006218&r2=1006219&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Sat Oct 9 19:23:36 2010
@@ -40,7 +40,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -181,7 +180,7 @@ public class HConnectionManager {
}
/* Encapsulates connection to zookeeper and regionservers.*/
- static class HConnectionImplementation implements HConnection, Abortable {
+ static class HConnectionImplementation implements HConnection {
static final Log LOG = LogFactory.getLog(HConnectionImplementation.class);
private final Class<? extends HRegionInterface> serverInterfaceClass;
private final long pause;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java?rev=1006219&r1=1006218&r2=1006219&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HMerge.java Sat Oct 9 19:23:36 2010
@@ -154,7 +154,7 @@ class HMerge {
void process() throws IOException {
try {
- for(HRegionInfo[] regionsToMerge = next();
+ for (HRegionInfo[] regionsToMerge = next();
regionsToMerge != null;
regionsToMerge = next()) {
if (!merge(regionsToMerge)) {
@@ -172,7 +172,7 @@ class HMerge {
}
protected boolean merge(final HRegionInfo[] info) throws IOException {
- if(info.length < 2) {
+ if (info.length < 2) {
LOG.info("only one region - nothing to merge");
return false;
}
@@ -196,8 +196,8 @@ class HMerge {
if ((currentSize + nextSize) <= (maxFilesize / 2)) {
// We merge two adjacent regions if their total size is less than
// one half of the desired maximum size
- LOG.info("merging regions " + Bytes.toString(currentRegion.getRegionName())
- + " and " + Bytes.toString(nextRegion.getRegionName()));
+ LOG.info("Merging regions " + currentRegion.getRegionNameAsString() +
+ " and " + nextRegion.getRegionNameAsString());
HRegion mergedRegion =
HRegion.mergeAdjacent(currentRegion, nextRegion);
updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1006219&r1=1006218&r2=1006219&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Sat Oct 9 19:23:36 2010
@@ -214,6 +214,11 @@ public class HBaseTestingUtility {
System.setProperty("test.cache.data", this.clusterTestBuildDir.toString());
this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, true,
true, null, null, null, null);
+ // Set this just-started cluser as our filesystem.
+ FileSystem fs = this.dfsCluster.getFileSystem();
+ this.conf.set("fs.defaultFS", fs.getUri().toString());
+ // Do old style too just to be safe.
+ this.conf.set("fs.default.name", fs.getUri().toString());
return this.dfsCluster;
}
@@ -318,7 +323,7 @@ public class HBaseTestingUtility {
// If we already put up a cluster, fail.
String testBuildPath = conf.get(TEST_DIRECTORY_KEY, null);
isRunningCluster(testBuildPath);
- if(testBuildPath != null) {
+ if (testBuildPath != null) {
LOG.info("Using passed path: " + testBuildPath);
}
// Make a new random dir to home everything in. Set it as system property.
@@ -329,24 +334,30 @@ public class HBaseTestingUtility {
// Bring up mini dfs cluster. This spews a bunch of warnings about missing
// scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'.
startMiniDFSCluster(numSlaves, this.clusterTestBuildDir);
-
- // Mangle conf so fs parameter points to minidfs we just started up
- FileSystem fs = this.dfsCluster.getFileSystem();
- this.conf.set("fs.defaultFS", fs.getUri().toString());
- // Do old style too just to be safe.
- this.conf.set("fs.default.name", fs.getUri().toString());
this.dfsCluster.waitClusterUp();
// Start up a zk cluster.
if (this.zkCluster == null) {
startMiniZKCluster(this.clusterTestBuildDir);
}
+ return startMiniHBaseCluster(numMasters, numSlaves);
+ }
+ /**
+ * Starts up mini hbase cluster. Usually used after call to
+ * {@link #startMiniCluster(int, int)} when doing stepped startup of clusters.
+ * Usually you won't want this. You'll usually want {@link #startMiniCluster()}.
+ * @param numMasters
+ * @param numSlaves
+ * @return Reference to the hbase mini hbase cluster.
+ * @throws IOException
+ * @see {@link #startMiniCluster()}
+ */
+ public MiniHBaseCluster startMiniHBaseCluster(final int numMasters,
+ final int numSlaves)
+ throws IOException {
// Now do the mini hbase cluster. Set the hbase.rootdir in config.
- Path hbaseRootdir = fs.makeQualified(fs.getHomeDirectory());
- this.conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
- fs.mkdirs(hbaseRootdir);
- FSUtils.setVersion(fs, hbaseRootdir);
+ createRootDir();
Configuration c = new Configuration(this.conf);
this.hbaseCluster = new MiniHBaseCluster(c, numMasters, numSlaves);
// Don't leave here till we've done a successful scan of the .META.
@@ -386,6 +397,7 @@ public class HBaseTestingUtility {
}
/**
+ * Stops mini hbase, zk, and hdfs clusters.
* @throws IOException
* @see {@link #startMiniCluster(int)}
*/
@@ -414,6 +426,23 @@ public class HBaseTestingUtility {
}
/**
+ * Creates an hbase rootdir in user home directory. Also creates hbase
+ * version file. Normally you won't make use of this method. Root hbasedir
+ * is created for you as part of mini cluster startup. You'd only use this
+ * method if you were doing manual operation.
+ * @return Fully qualified path to hbase root dir
+ * @throws IOException
+ */
+ public Path createRootDir() throws IOException {
+ FileSystem fs = FileSystem.get(this.conf);
+ Path hbaseRootdir = fs.makeQualified(fs.getHomeDirectory());
+ this.conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
+ fs.mkdirs(hbaseRootdir);
+ FSUtils.setVersion(fs, hbaseRootdir);
+ return hbaseRootdir;
+ }
+
+ /**
* Flushes all caches in the mini hbase cluster
* @throws IOException
*/
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java?rev=1006219&r1=1006218&r2=1006219&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java Sat Oct 9 19:23:36 2010
@@ -19,27 +19,145 @@
*/
package org.apache.hadoop.hbase.util;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
+import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.AbstractMergeTestBase;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.catalog.MetaReader;
import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.HMerge;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.junit.Test;
/**
* Tests merging a normal table's regions
*/
-public class TestMergeTable extends AbstractMergeTestBase {
+public class TestMergeTable {
+ private static final Log LOG = LogFactory.getLog(TestMergeTable.class);
+ private final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static final byte [] COLUMN_NAME = Bytes.toBytes("contents");
+ private static final byte [] VALUE;
+ static {
+ // We will use the same value for the rows as that is not really important here
+ String partialValue = String.valueOf(System.currentTimeMillis());
+ StringBuilder val = new StringBuilder();
+ while (val.length() < 1024) {
+ val.append(partialValue);
+ }
+ VALUE = Bytes.toBytes(val.toString());
+ }
/**
- * Test case
- * @throws IOException
+ * Test merge.
+ * Hand-makes regions of a mergeable size and adds the hand-made regions to
+ * hand-made meta. The hand-made regions are created offline. We then start
+ * up mini cluster, disables the hand-made table and starts in on merging.
+ * @throws Exception
*/
- public void testMergeTable() throws IOException {
- assertNotNull(dfsCluster);
- Configuration c = new Configuration(this.conf);
- HBaseAdmin admin = new HBaseAdmin(c);
- admin.disableTable(desc.getName());
- HMerge.merge(c, dfsCluster.getFileSystem(), desc.getName());
+ @Test public void testMergeTable() throws Exception {
+ // Table we are manually creating offline.
+ HTableDescriptor desc = new HTableDescriptor(Bytes.toBytes("test"));
+ desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
+
+ // Set maximum regionsize down.
+ UTIL.getConfiguration().setLong("hbase.hregion.max.filesize", 64L * 1024L * 1024L);
+ // Startup hdfs. Its in here we'll be putting our manually made regions.
+ UTIL.startMiniDFSCluster(1);
+ // Create hdfs hbase rootdir.
+ Path rootdir = UTIL.createRootDir();
+
+ // Now create three data regions: The first is too large to merge since it
+ // will be > 64 MB in size. The second two will be smaller and will be
+ // selected for merging.
+
+ // To ensure that the first region is larger than 64MB we need to write at
+ // least 65536 rows. We will make certain by writing 70000
+ byte [] row_70001 = Bytes.toBytes("row_70001");
+ byte [] row_80001 = Bytes.toBytes("row_80001");
+
+ // Create regions and populate them at same time.
+ HRegion [] regions = {
+ createRegion(desc, null, row_70001, 1, 70000, rootdir),
+ createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir),
+ createRegion(desc, row_80001, null, 80001, 11000, rootdir)
+ };
+
+ // Now create the root and meta regions and insert the data regions
+ // created above into .META.
+ setupROOTAndMeta(rootdir, regions);
+ try {
+ LOG.info("Starting mini zk cluster");
+ UTIL.startMiniZKCluster();
+ LOG.info("Starting mini hbase cluster");
+ UTIL.startMiniHBaseCluster(1, 1);
+ Configuration c = new Configuration(UTIL.getConfiguration());
+ HConnection connection = HConnectionManager.getConnection(c);
+ CatalogTracker ct = new CatalogTracker(connection);
+ ct.start();
+ List<HRegionInfo> originalTableRegions =
+ MetaReader.getTableRegions(ct, desc.getName());
+ LOG.info("originalTableRegions size=" + originalTableRegions.size() +
+ "; " + originalTableRegions);
+ HBaseAdmin admin = new HBaseAdmin(new Configuration(c));
+ admin.disableTable(desc.getName());
+ HMerge.merge(c, FileSystem.get(c), desc.getName());
+ List<HRegionInfo> postMergeTableRegions =
+ MetaReader.getTableRegions(ct, desc.getName());
+ LOG.info("postMergeTableRegions size=" + postMergeTableRegions.size() +
+ "; " + postMergeTableRegions);
+ assertTrue(postMergeTableRegions.size() < originalTableRegions.size());
+ } finally {
+ UTIL.shutdownMiniCluster();
+ }
+ }
+
+ private HRegion createRegion(final HTableDescriptor desc,
+ byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir)
+ throws IOException {
+ HRegionInfo hri = new HRegionInfo(desc, startKey, endKey);
+ HRegion region = HRegion.createHRegion(hri, rootdir, UTIL.getConfiguration());
+ LOG.info("Created region " + region.getRegionNameAsString());
+ for(int i = firstRow; i < firstRow + nrows; i++) {
+ Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i)));
+ put.add(COLUMN_NAME, null, VALUE);
+ region.put(put);
+ if (i % 10000 == 0) {
+ LOG.info("Flushing write #" + i);
+ region.flushcache();
+ }
+ }
+ region.close();
+ region.getLog().closeAndDelete();
+ return region;
+ }
+
+ protected void setupROOTAndMeta(Path rootdir, final HRegion [] regions)
+ throws IOException {
+ HRegion root =
+ HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, rootdir, UTIL.getConfiguration());
+ HRegion meta =
+ HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, rootdir,
+ UTIL.getConfiguration());
+ HRegion.addRegionToMETA(root, meta);
+ for (HRegion r: regions) {
+ HRegion.addRegionToMETA(meta, r);
+ }
+ meta.close();
+ meta.getLog().closeAndDelete();
+ root.close();
+ root.getLog().closeAndDelete();
}
}
\ No newline at end of file