You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ji...@apache.org on 2008/02/13 01:09:54 UTC
svn commit: r627195 - in /hadoop/hbase/trunk: CHANGES.txt
src/test/org/apache/hadoop/hbase/MultiRegionTable.java
src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java
src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
Author: jimk
Date: Tue Feb 12 16:09:52 2008
New Revision: 627195
URL: http://svn.apache.org/viewvc?rev=627195&view=rev
Log:
HBASE-43[45] TestTableIndex and TestTableMapReduce failed in Hudson builds
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=627195&r1=627194&r2=627195&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Tue Feb 12 16:09:52 2008
@@ -18,6 +18,7 @@
HBASE-421 TestRegionServerExit broken
HBASE-426 hbase can't find remote filesystem
HBASE-437 Clear Command should use system.out (Edward Yoon via Stack)
+ HBASE-43[45] TestTableIndex and TestTableMapReduce failed in Hudson builds
IMPROVEMENTS
HBASE-415 Rewrite leases to use DelayedBlockingQueue instead of polling
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MultiRegionTable.java?rev=627195&r1=627194&r2=627195&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/MultiRegionTable.java Tue Feb 12 16:09:52 2008
@@ -88,11 +88,17 @@
// with EMPTY_START_ROW will be one of the unsplittable daughters.
HRegionInfo hri = null;
HRegion r = null;
+ HRegionServer server = cluster.getRegionThreads().get(0).getRegionServer();
for (int i = 0; i < 30; i++) {
- hri = t.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo();
+ try {
+ hri = t.getRegionLocation(HConstants.EMPTY_START_ROW).getRegionInfo();
+ } catch (IOException e) {
+ e = RemoteExceptionHandler.checkIOException(e);
+ e.printStackTrace();
+ continue;
+ }
LOG.info("Region location: " + hri);
- r = cluster.getRegionThreads().get(0).getRegionServer().
- onlineRegions.get(hri.getRegionName());
+ r = server.onlineRegions.get(hri.getRegionName());
if (r != null) {
break;
}
@@ -102,10 +108,10 @@
LOG.warn("Waiting on region to come online", e);
}
}
+ assertNotNull(r);
// Flush the cache
- cluster.getRegionThreads().get(0).getRegionServer().getCacheFlushListener().
- flushRequested(r);
+ server.getCacheFlushListener().flushRequested(r);
// Now, wait until split makes it into the meta table.
int oldCount = count;
@@ -158,7 +164,8 @@
// still has references.
while (true) {
data = getSplitParentInfo(meta, parent);
- if (data == null || data.size() == 3) {
+ if (data != null && data.size() == 3) {
+ LOG.info("Waiting for splitA to release reference to parent");
try {
Thread.sleep(waitTime);
} catch (InterruptedException e) {
@@ -168,7 +175,9 @@
}
break;
}
- LOG.info("Parent split info returned " + data.keySet().toString());
+ if (data != null) {
+ LOG.info("Parent split info returned " + data.keySet().toString());
+ }
}
if (splitB == null) {
@@ -199,8 +208,10 @@
for (int i = 0; i < retries; i++) {
if (!fs.exists(parentDir)) {
+ LOG.info("Parent directory was deleted. tries=" + i);
break;
}
+ LOG.info("Waiting for parent directory to be deleted. tries=" + i);
try {
Thread.sleep(waitTime);
} catch (InterruptedException e) {
@@ -260,8 +271,7 @@
continue;
}
// Make sure I get the parent.
- if (hri.getRegionName().toString().
- equals(parent.getRegionName().toString()) &&
+ if (hri.getRegionName().equals(parent.getRegionName()) &&
hri.getRegionId() == parent.getRegionId()) {
return curVals;
}
@@ -316,8 +326,7 @@
* @throws IOException
*/
protected static void compact(final MiniHBaseCluster cluster,
- final HRegionInfo r)
- throws IOException {
+ final HRegionInfo r) throws IOException {
if (r == null) {
LOG.debug("Passed region is null");
return;
@@ -332,8 +341,7 @@
for (int i = 0; i < 10; i++) {
try {
for (HRegion online: regions.values()) {
- if (online.getRegionName().toString().
- equals(r.getRegionName().toString())) {
+ if (online.getRegionName().equals(r.getRegionName())) {
online.compactStores();
}
}
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java?rev=627195&r1=627194&r2=627195&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableIndex.java Tue Feb 12 16:09:52 2008
@@ -31,6 +31,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.dfs.MiniDFSCluster;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseAdmin;
@@ -96,6 +97,9 @@
// below. After adding all data, the first region is 1.3M
conf.setLong("hbase.hregion.max.filesize", 1024 * 1024);
+ // Always compact if there is more than one store file.
+ conf.setInt("hbase.hstore.compactionThreshold", 2);
+
desc = new HTableDescriptor(TABLE_NAME);
desc.addFamily(new HColumnDescriptor(INPUT_COLUMN));
desc.addFamily(new HColumnDescriptor(OUTPUT_COLUMN));
@@ -160,7 +164,6 @@
}
scanTable(printResults);
- @SuppressWarnings("deprecation")
MiniMRCluster mrCluster = new MiniMRCluster(2, fs.getUri().toString(), 1);
// set configuration parameter for index build
@@ -267,17 +270,17 @@
Integer.toString(new Random().nextInt()));
this.fs.copyToLocalFile(new Path(INDEX_DIR), localDir);
FileSystem localfs = FileSystem.getLocal(conf);
- Path [] indexDirs = localfs.listPaths(new Path [] {localDir});
+ FileStatus [] indexDirs = localfs.listStatus(localDir);
Searcher searcher = null;
HScannerInterface scanner = null;
try {
if (indexDirs.length == 1) {
- searcher = new IndexSearcher((new File(indexDirs[0].
+ searcher = new IndexSearcher((new File(indexDirs[0].getPath().
toUri())).getAbsolutePath());
} else if (indexDirs.length > 1) {
Searchable[] searchers = new Searchable[indexDirs.length];
for (int i = 0; i < indexDirs.length; i++) {
- searchers[i] = new IndexSearcher((new File(indexDirs[i].
+ searchers[i] = new IndexSearcher((new File(indexDirs[i].getPath().
toUri()).getAbsolutePath()));
}
searcher = new MultiSearcher(searchers);
@@ -301,7 +304,6 @@
int count = 0;
while (scanner.next(key, results)) {
String value = key.getRow().toString();
- LOG.debug("Scanned over " + key.getRow());
Term term = new Term(rowkeyName, value);
int hitCount = searcher.search(new TermQuery(term)).length();
assertEquals("check row " + value, 1, hitCount);
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java?rev=627195&r1=627194&r2=627195&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/mapred/TestTableMapReduce.java Tue Feb 12 16:09:52 2008
@@ -38,6 +38,7 @@
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.MultiRegionTable;
import org.apache.hadoop.hbase.StaticTestEnvironment;
+import org.apache.hadoop.hbase.io.BatchUpdate;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
@@ -99,7 +100,7 @@
// This size should make it so we always split using the addContent
// below. After adding all data, the first region is 1.3M
- conf.setLong("hbase.hregion.max.filesize", 256 * 1024);
+ conf.setLong("hbase.hregion.max.filesize", 1024 * 1024);
// Make lease timeout longer, lease checks less frequent
conf.setInt("hbase.master.lease.period", 10 * 1000);
@@ -156,7 +157,7 @@
/**
* Pass the key, and reversed value to reduce
*
- * @see org.apache.hadoop.hbase.mapred.TableMap#map(org.apache.hadoop.hbase.HStoreKey, org.apache.hadoop.io.MapWritable, org.apache.hadoop.hbase.mapred.TableOutputCollector, org.apache.hadoop.mapred.Reporter)
+ * @see org.apache.hadoop.hbase.mapred.TableMap#map(org.apache.hadoop.hbase.HStoreKey, org.apache.hadoop.io.MapWritable, org.apache.hadoop.mapred.OutputCollector, org.apache.hadoop.mapred.Reporter)
*/
@SuppressWarnings("unchecked")
@Override
@@ -222,17 +223,11 @@
HTable table = new HTable(conf, new Text(SINGLE_REGION_TABLE_NAME));
for(int i = 0; i < values.length; i++) {
- long lockid = table.startUpdate(new Text("row_"
- + String.format("%1$05d", i)));
+ BatchUpdate b = new BatchUpdate(new Text("row_" +
+ String.format("%1$05d", i)));
- try {
- table.put(lockid, TEXT_INPUT_COLUMN, values[i]);
- table.commit(lockid, System.currentTimeMillis());
- lockid = -1;
- } finally {
- if (lockid != -1)
- table.abort(lockid);
- }
+ b.put(TEXT_INPUT_COLUMN, values[i]);
+ table.commit(b);
}
LOG.info("Print table contents before map/reduce for " +