You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2007/11/09 22:57:25 UTC
svn commit: r593665 - in /lucene/hadoop/trunk/src/contrib/hbase: ./
src/java/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/
Author: stack
Date: Fri Nov 9 13:57:24 2007
New Revision: 593665
URL: http://svn.apache.org/viewvc?rev=593665&view=rev
Log:
HADOOP-2161 getRow() is orders of magnitudes slower than get(), even on
rows with one column
Modified:
lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
Modified: lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/CHANGES.txt Fri Nov 9 13:57:24 2007
@@ -28,6 +28,8 @@
problematic.
HADOOP-2155 Method expecting HBaseConfiguration throws NPE when given Configuration
HADOOP-2156 BufferUnderflowException for un-named HTableDescriptors
+ HADOOP-2161 getRow() is orders of magnitudes slower than get(), even on rows
+ with one column (Clint Morgan and Stack)
IMPROVEMENTS
HADOOP-2401 Add convenience put method that takes writable
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMaster.java Fri Nov 9 13:57:24 2007
@@ -3063,7 +3063,7 @@
(new LocalHBaseCluster(conf)).startup();
} else {
Constructor<? extends HMaster> c =
- masterClass.getConstructor(Configuration.class);
+ masterClass.getConstructor(HBaseConfiguration.class);
HMaster master = c.newInstance(conf);
master.start();
}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HMemcache.java Fri Nov 9 13:57:24 2007
@@ -235,7 +235,7 @@
&& key.matchesWithoutColumn(itKey)) {
byte [] val = tailMap.get(itKey);
results.put(itCol, val);
- } else if (key.getRow().compareTo(itKey.getRow()) > 0) {
+ } else if (key.getRow().compareTo(itKey.getRow()) < 0) {
break;
}
}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Fri Nov 9 13:57:24 2007
@@ -35,7 +35,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.filter.RowFilterInterface;
@@ -1018,13 +1017,11 @@
*/
TreeMap<Text, byte []> getFull(Text row) throws IOException {
HStoreKey key = new HStoreKey(row, System.currentTimeMillis());
-
lock.obtainReadLock();
try {
TreeMap<Text, byte []> memResult = memcache.getFull(key);
for (Text colFamily: stores.keySet()) {
- HStore targetStore = stores.get(colFamily);
- targetStore.getFull(key, memResult);
+ this.stores.get(colFamily).getFull(key, memResult);
}
return memResult;
} finally {
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Fri Nov 9 13:57:24 2007
@@ -1453,7 +1453,7 @@
"hbase.master is set to 'local' mode");
} else {
Constructor<? extends HRegionServer> c =
- regionServerClass.getConstructor(Configuration.class);
+ regionServerClass.getConstructor(HBaseConfiguration.class);
HRegionServer hrs = c.newInstance(conf);
Thread t = new Thread(hrs);
t.setName("regionserver" + hrs.server.getListenerAddress());
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HScannerInterface.java Fri Nov 9 13:57:24 2007
@@ -21,6 +21,7 @@
import java.io.Closeable;
import java.io.IOException;
+import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
@@ -28,7 +29,8 @@
/**
* HScannerInterface iterates through a set of rows. It's implemented by
- * several classes.
+ * several classes. Implements {@link Iterable} but be sure to still call
+ * {@link #close()} when done with your {@link Iterator}
*/
public interface HScannerInterface extends Closeable,
Iterable<Map.Entry<HStoreKey, SortedMap<Text, byte []>>> {
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Fri Nov 9 13:57:24 2007
@@ -1100,7 +1100,7 @@
}
results.put(new Text(readcol), readval.get());
readval = new ImmutableBytesWritable();
- } else if(key.getRow().compareTo(readkey.getRow()) > 0) {
+ } else if(key.getRow().compareTo(readkey.getRow()) < 0) {
break;
}
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java Fri Nov 9 13:57:24 2007
@@ -564,13 +564,19 @@
/**
* Start an atomic row insertion/update. No changes are committed until the
- * call to commit() returns.
- *
- * A call to abort() will abandon any updates in progress.
+ * call to commit() returns. A call to abort() will abandon any updates in
+ * progress.
*
*
- * @param row Name of row to start update against.
- * @return Row lockid.
+ * @param row Name of row to start update against. Note, choose row names
+ * with care. Rows are sorted lexicographically (comparison is done
+ * using {@link Text#compareTo(Object)}. If your keys are numeric,
+ * lexicographic sorting means that 46 sorts AFTER 450 (If you want to use
+ * numerics for keys, zero-pad).
+ * @return Row lock id..
+ * @see #commit(long)
+ * @see #commit(long, long)
+ * @see #abort(long)
*/
public synchronized long startUpdate(final Text row) {
checkClosed();
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Fri Nov 9 13:57:24 2007
@@ -37,6 +37,8 @@
protected final static String COLFAMILY_NAME1 = "colfamily1:";
protected final static String COLFAMILY_NAME2 = "colfamily2:";
protected final static String COLFAMILY_NAME3 = "colfamily3:";
+ protected static Text [] COLUMNS = new Text [] {new Text(COLFAMILY_NAME1),
+ new Text(COLFAMILY_NAME2), new Text(COLFAMILY_NAME3)};
protected Path testDir = null;
protected FileSystem localFs = null;
protected static final char FIRST_CHAR = 'a';
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java?rev=593665&r1=593664&r2=593665&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/PerformanceEvaluation.java Fri Nov 9 13:57:24 2007
@@ -25,6 +25,7 @@
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
+import java.util.Formatter;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;
@@ -293,6 +294,7 @@
protected HBaseAdmin admin;
protected HTable table;
protected volatile HBaseConfiguration conf;
+ private Formatter formatter = new Formatter();
Test(final HBaseConfiguration conf, final int startRow,
final int perClientRunRows, final int totalRows, final Status status) {
@@ -366,10 +368,14 @@
}
Text getRandomRow() {
- return new Text(Integer.toString(this.rand.nextInt(Integer.MAX_VALUE) %
+ return new Text(format(this.rand.nextInt(Integer.MAX_VALUE) %
this.totalRows));
}
+ public Text format(final int i) {
+ return new Text(String.format("%010d", Integer.valueOf(i)));
+ }
+
/*
* Test for individual row.
* @param i Row index.
@@ -439,7 +445,7 @@
void testSetup() throws IOException {
super.testSetup();
this.testScanner = table.obtainScanner(new Text[] {COLUMN_NAME},
- new Text(Integer.toString(this.startRow)));
+ format(this.startRow));
}
@Override
@@ -471,7 +477,7 @@
@Override
void testRow(final int i) throws IOException {
- table.get(new Text(Integer.toString(i)), COLUMN_NAME);
+ table.get(format(i), COLUMN_NAME);
}
@Override
@@ -488,7 +494,7 @@
@Override
void testRow(final int i) throws IOException {
- long lockid = table.startUpdate(new Text(Integer.toString(i)));
+ long lockid = table.startUpdate(format(i));
table.put(lockid, COLUMN_NAME, generateValue());
table.commit(lockid);
}